diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 0000000..86eb663 --- /dev/null +++ b/.bazelignore @@ -0,0 +1,4 @@ +# Using CMake or own WORKSPACE +oss-internship-2020 +# Uses its own WORKSPACE +sandboxed_api/examples/hello_sapi diff --git a/.gitmodules b/.gitmodules index 1e4c43c..ec36987 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,9 @@ [submodule "oss-internship-2020/pffft/master"] path = oss-internship-2020/pffft/master url = https://bitbucket.org/jpommier/pffft/src/master/ +[submodule "oss-internship-2020/curl/curl_wrapper/curl"] + path = oss-internship-2020/curl/curl_wrapper/curl + url = https://github.com/curl/curl +[submodule "oss-internship-2020/gdal/gdal"] + path = oss-internship-2020/gdal/gdal + url = https://github.com/OSGeo/gdal/tree/master/gdal diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c993f4..6d141be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,16 @@ cmake_minimum_required(VERSION 3.12) +# Fix Ninja generator output to not rebuild entire sub-trees needlessly. +if(CMAKE_GENERATOR MATCHES "Ninja") + file(WRITE "${CMAKE_BINARY_DIR}/UserMakeRulesOverride.cmake" + "string(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_C \"\${CMAKE_DEPFILE_FLAGS_C}\")\n" + "string(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_CXX \"\${CMAKE_DEPFILE_FLAGS_CXX}\")\n" + ) + set(CMAKE_USER_MAKE_RULES_OVERRIDE + "${CMAKE_BINARY_DIR}/UserMakeRulesOverride.cmake" CACHE INTERNAL "") +endif() + project(SandboxedAPI C CXX ASM) # SAPI-wide setting for the language level @@ -31,15 +41,11 @@ include(SapiDeps) include(SapiUtil) include(SapiBuildDefs) -# Fix Ninja generator output to not rebuild entire sub-trees needlessly. -if(CMAKE_GENERATOR MATCHES "Ninja") - file(WRITE "${SAPI_BINARY_DIR}/UserMakeRulesOverride.cmake" - "STRING(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_C \"\${CMAKE_DEPFILE_FLAGS_C}\")\n" - "STRING(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_CXX \"\${CMAKE_DEPFILE_FLAGS_CXX}\")\n" - ) - set(CMAKE_USER_MAKE_RULES_OVERRIDE - "${SAPI_BINARY_DIR}/UserMakeRulesOverride.cmake" CACHE INTERNAL "") -endif() +# Allow the header generator to auto-configure include paths +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# Allow the header generator to auto-configure include paths +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if (SAPI_FORCE_COLOR_OUTPUT) if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # GCC diff --git a/cmake/SapiBuildDefs.cmake b/cmake/SapiBuildDefs.cmake index 62583ec..cd0e43d 100644 --- a/cmake/SapiBuildDefs.cmake +++ b/cmake/SapiBuildDefs.cmake @@ -123,7 +123,7 @@ function(add_sapi_library) endif() # Interface - list_join(_sapi_FUNCTIONS "," _sapi_funcs) + list(JOIN _sapi_FUNCTIONS "," _sapi_funcs) foreach(src IN LISTS _sapi_INPUTS) get_filename_component(src "${src}" ABSOLUTE) list(APPEND _sapi_full_inputs "${src}") @@ -132,44 +132,44 @@ function(add_sapi_library) set(_sapi_embed_dir "${CMAKE_CURRENT_BINARY_DIR}") set(_sapi_embed_name "${_sapi_NAME}") endif() + + set(_sapi_isystem "${_sapi_NAME}.isystem}") + list(APPEND _sapi_generator_args + "--sapi_name=${_sapi_LIBRARY_NAME}" + "--sapi_out=${_sapi_gen_header}" + "--sapi_embed_dir=${_sapi_embed_dir}" + "--sapi_embed_name=${_sapi_embed_name}" + "--sapi_functions=${_sapi_funcs}" + "--sapi_ns=${_sapi_NAMESPACE}" + "--sapi_isystem=${_sapi_isystem}" + ) + list(JOIN _sapi_full_inputs "," _sapi_full_inputs) if(SAPI_ENABLE_GENERATOR) - add_custom_command( - OUTPUT "${_sapi_gen_header}" - COMMAND sapi_generator_tool - "--sapi_name=${_sapi_LIBRARY_NAME}" - "--sapi_out=${_sapi_gen_header}" - "--sapi_embed_dir=${_sapi_embed_dir}" - "--sapi_embed_name=${_sapi_embed_name}" - "--sapi_functions=${_sapi_funcs}" - "--sapi_ns=${_sapi_NAMESPACE}" - ${_sapi_full_inputs} - COMMENT "Generating interface" - DEPENDS ${_sapi_INPUTS} - VERBATIM + list(APPEND _sapi_generator_command + sapi_generator_tool + -p "${CMAKE_CURRENT_BINARY_DIR}" + ${_sapi_generator_args} + ${_sapi_full_inputs} ) else() - set(_sapi_isystem "${_sapi_NAME}.isystem") - list_join(_sapi_full_inputs "," _sapi_full_inputs) - add_custom_command( - OUTPUT "${_sapi_gen_header}" "${_sapi_isystem}" - COMMAND sh -c - "${CMAKE_CXX_COMPILER} -E -x c++ -v /dev/null 2>&1 | \ - awk '/> search starts here:/{f=1;next}/^End of search/{f=0}f{print $1}' \ - > \"${_sapi_isystem}\"" - COMMAND "${SAPI_PYTHON3_EXECUTABLE}" -B - "${SAPI_SOURCE_DIR}/sandboxed_api/tools/generator2/sapi_generator.py" - "--sapi_name=${_sapi_LIBRARY_NAME}" - "--sapi_out=${_sapi_gen_header}" - "--sapi_embed_dir=${_sapi_embed_dir}" - "--sapi_embed_name=${_sapi_embed_name}" - "--sapi_functions=${_sapi_funcs}" - "--sapi_ns=${_sapi_NAMESPACE}" - "--sapi_isystem=${_sapi_isystem}" - "--sapi_in=${_sapi_full_inputs}" - COMMENT "Generating interface" - VERBATIM + list(APPEND _sapi_generator_command + "${SAPI_PYTHON3_EXECUTABLE}" -B + "${SAPI_SOURCE_DIR}/sandboxed_api/tools/generator2/sapi_generator.py" + ${_sapi_generator_args} + "--sapi_in=${_sapi_full_inputs}" ) endif() + add_custom_command( + OUTPUT "${_sapi_gen_header}" "${_sapi_isystem}" + COMMAND sh -c + "${CMAKE_CXX_COMPILER} -E -x c++ -v /dev/null 2>&1 | \ + awk '/> search starts here:/{f=1;next}/^End of search/{f=0}f{print $1}' \ + > \"${_sapi_isystem}\"" + COMMAND ${_sapi_generator_command} + COMMENT "Generating interface" + DEPENDS ${_sapi_INPUTS} + VERBATIM + ) # Library with the interface if(NOT _sapi_SOURCES) diff --git a/cmake/SapiUtil.cmake b/cmake/SapiUtil.cmake index 8773f59..f4e891f 100644 --- a/cmake/SapiUtil.cmake +++ b/cmake/SapiUtil.cmake @@ -35,16 +35,6 @@ function(create_directory_symlink SOURCE DESTINATION) endif() endfunction() -# Implements list(JOIN ...) for CMake < 3.12. -function(list_join LIST SEP OUTPUT) - foreach(item IN LISTS ${LIST}) - set(_concat "${_concat}${SEP}${item}") - endforeach() - string(LENGTH "${SEP}" _len) - string(SUBSTRING "${_concat}" ${_len} -1 _concat) - set(${OUTPUT} "${_concat}" PARENT_SCOPE) -endfunction() - # Helper function that behaves just like Protobuf's protobuf_generate_cpp(), # except that it strips import paths. This is necessary, because CMake's # protobuf rules don't work well with imports across different directories. diff --git a/oss-internship-2020/curl/.gitignore b/oss-internship-2020/curl/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/oss-internship-2020/curl/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/oss-internship-2020/curl/CMakeLists.txt b/oss-internship-2020/curl/CMakeLists.txt new file mode 100644 index 0000000..8d2be38 --- /dev/null +++ b/oss-internship-2020/curl/CMakeLists.txt @@ -0,0 +1,144 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16) + +project(libcurl_sandbox) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +option(CURL_SAPI_ENABLE_EXAMPLES "" ON) +option(CURL_SAPI_ENABLE_TESTS "" ON) + +# Add callbacks used by examples and tests +if (CURL_SAPI_ENABLE_EXAMPLES OR CURL_SAPI_ENABLE_TESTS) + list(APPEND CURL_SAPI_CALLBACKS + "${CMAKE_CURRENT_SOURCE_DIR}/callbacks/callbacks.h" + "${CMAKE_CURRENT_SOURCE_DIR}/callbacks/callbacks.cc" + ) +endif() + +# Add folder containing the non-sandboxed custom curl library +add_subdirectory(curl_wrapper) + +# Setup Sandboxed API +set(SAPI_ROOT "" CACHE PATH "Path to the Sandboxed API source tree") +set(SAPI_ENABLE_EXAMPLES ${CURL_SAPI_ENABLE_EXAMPLES} CACHE BOOL "" FORCE) +set(SAPI_ENABLE_TESTS ${CURL_SAPI_ENABLE_TESTS} CACHE BOOL "" FORCE) +add_subdirectory( + "${SAPI_ROOT}" + "${CMAKE_BINARY_DIR}/sandboxed-api-build" + EXCLUDE_FROM_ALL +) + +# Generate SAPI header +add_sapi_library(curl_sapi + + # List of all the methods in https://curl.haxx.se/libcurl/c/allfuncs.html + # Some are added or modified because the original ones are not supported + # by Sandboxed API (details can be found in curl_wrapper.h) + FUNCTIONS curl_easy_cleanup + curl_easy_duphandle + curl_easy_escape + curl_easy_getinfo + curl_easy_getinfo_ptr + curl_easy_init + curl_easy_pause + curl_easy_perform + curl_easy_recv + curl_easy_reset + curl_easy_send + curl_easy_setopt + curl_easy_setopt_ptr + curl_easy_setopt_long + curl_easy_setopt_curl_off_t + curl_easy_strerror + curl_easy_unescape + curl_easy_upkeep + curl_free + curl_getdate_sapi + curl_global_cleanup + curl_global_init + curl_global_init_mem + curl_global_sslset + curl_mime_addpart + curl_mime_data + curl_mime_data_cb + curl_mime_encoder + curl_mime_filedata + curl_mime_filename + curl_mime_free + curl_mime_headers + curl_mime_init + curl_mime_name + curl_mime_subparts + curl_mime_type + curl_multi_add_handle + curl_multi_assign + curl_multi_cleanup + curl_multi_fdset_sapi + curl_multi_info_read + curl_multi_init + curl_multi_perform + curl_multi_remove_handle + curl_multi_setopt + curl_multi_setopt_ptr + curl_multi_setopt_long + curl_multi_setopt_curl_off_t + curl_multi_socket_action + curl_multi_strerror + curl_multi_timeout + curl_multi_poll_sapi + curl_multi_wait_sapi + curl_multi_wakeup + curl_share_init + curl_share_setopt + curl_share_setopt_ptr + curl_share_setopt_long + curl_share_strerror + curl_slist_append + curl_slist_free_all + curl_url + curl_url_cleanup + curl_url_dup + curl_url_get + curl_url_set + curl_version + curl_version_info + + INPUTS curl_wrapper/curl/include/curl/curl.h + curl_wrapper/curl_wrapper.h + + LIBRARY curl_wrapper_and_callbacks + + LIBRARY_NAME Curl + + NAMESPACE "" +) + +# Include generated SAPI header +target_include_directories(curl_sapi INTERFACE + "${PROJECT_BINARY_DIR}" +) + +# Add examples +if (CURL_SAPI_ENABLE_EXAMPLES) + add_subdirectory(examples) +endif() + +# Add tests +if (CURL_SAPI_ENABLE_TESTS) + add_subdirectory(tests) +endif() diff --git a/oss-internship-2020/curl/README.md b/oss-internship-2020/curl/README.md new file mode 100644 index 0000000..3000838 --- /dev/null +++ b/oss-internship-2020/curl/README.md @@ -0,0 +1,112 @@ +# LibCurl Sandbox + +This library is a sandboxed version of curl's C API, +[libcurl](https://curl.haxx.se/libcurl/c/), implemented using Sandboxed API. + +## Setup + +The repository can be cloned using: `git clone --recursive [URL to this repo]` +The `--recursive` flag ensures that submodules are also cloned. + +Alternatively, if the repository has already been cloned but the submodules have +not, these can be cloned using: `git submodule update --init --recursive` + +The full list of Sandboxed API dependencies can be found on +[Sandboxed API Getting Started page](https://developers.google.com/sandboxed-api/docs/getting-started). + +The following commands, used from the current `curl/` directory, build the +library: `mkdir -p build cd build cmake .. -G Ninja -D SAPI_ROOT=[path to +sandboxed-api] cmake --build .` + +## Implementation details + +All of libcurl's methods are supported by the library. However, a few of these +have different signatures defined in the sandboxed header `custom_curl.h`, which +wraps and extends libcurl. + +This is necessary because Sandboxed API sandboxes most of libcurl correctly, but +encounters some issues when sandboxing a few methods. The simplest solution is +wrapping these methods into wrapper methods that accomplish the same tasks but +can also be sandboxed. + +The next sections describe the issues encountered and contain some information +on the signatures of the wrapper methods solving these issues. + +#### Variadic methods + +Variadic methods are currently not supported by Sandboxed API. To solve this, +these methods are defined with an additional explicit parameter in +`custom_curl.h`. + +The methods are: - `curl_easy_setopt`. Use `curl_easy_setopt_ptr`, +`curl_easy_setopt_long` or `curl_easy_setopt_curl_off_t` instead. - +`curl_easy_getinfo`. Use `curl_easy_getinfo_ptr` instead. - `curl_multi_setopt`. +Use `curl_multi_setopt_ptr`, `curl_multi_setopt_long` or +`curl_multi_setopt_curl_off_t` instead. - `curl_share_setopt`. Use +`curl_share_setopt_ptr` or `curl_share_setopt_long` instead + +#### Methods with incomplete array arguments + +Incomplete array arguments are currently not supported by Sandboxed API. To +solve this, methods taking an incomplete array argument have a wrapper in +`custom_curl.h`, and take a pointer as the argument. + +The methods are: - `curl_multi_poll`. Use `curl_multi_poll_sapi` instead. - +`curl_multi_wait`. Use `curl_multi_wait_sapi` instead. + +#### Methods with conflicts on the generated header + +Some methods create conflicts on the generated header because of redefined +`#define` directives from files included by the header. To solve this, the +conflicting types and methods are redefined in `custom_curl.h`. + +The types are: - `time_t`. Use `time_t_sapi` instead. - `fd_set`. Use +`fd_set_sapi` instead. + +The methods are: - `curl_getdate`. Use `curl_getdate_sapi` instead. - +`curl_multi_fdset`. Use `curl_multi_fdset_sapi` instead. + +#### Function pointers + +The functions whose pointers will be passed to the library's methods +(*callbacks*) can't be implemented in the files making use of the library, but +must be in other files. These files must be compiled together with the library, +and this is done by adding their absolute path to the cmake variable +`CURL_SAPI_CALLBACKS`. + +The pointers can then be obtained using an `RPCChannel` object, as shown in +`example2.cc`. + +## Examples + +The `examples` directory contains the sandboxed versions of example source codes +taken from [this page](https://curl.haxx.se/libcurl/c/example.html) on curl's +website. More information about each example can be found in the examples' +[README](examples/README.md). + +To build these examples when building the library, the cmake variable +`CURL_SAPI_ENABLE_EXAMPLES` must be set to `ON`. This enables Sandboxed API +examples as well. + +## Policy + +The `sandbox.h` file contains a policy allowing all is necessary for libcurl to +perform simple requests. It is used by all the examples, except by example3. +This example needs some additional policies and files in its namespace (since it +uses HTTPS), and the file `example3.cc` shows how to easily extend an existing +policy. + +## Testing + +The `tests` folder contains some test cases created using Google Test. The class +`CurlTestUtils` is used to facilitate some tasks that all test cases need, +including the setup of a mock local server on which test requests are performed. + +To build these tests when building the library, the cmake variable +`CURL_SAPI_ENABLE_TESTS` must be set to `ON`. This enables Sandboxed API tests +as well. + +## Callbacks + +The `callbacks.h` and `callbacks.cc` files implement all the callbacks used by +examples and tests. diff --git a/oss-internship-2020/curl/callbacks/callbacks.cc b/oss-internship-2020/curl/callbacks/callbacks.cc new file mode 100644 index 0000000..015e66c --- /dev/null +++ b/oss-internship-2020/curl/callbacks/callbacks.cc @@ -0,0 +1,37 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "callbacks.h" // NOLINT(build/include) + +#include +#include + +#include "sandboxed_api/vars.h" + +size_t WriteToMemory(char* contents, size_t size, size_t num_bytes, + void* userp) { + size_t real_size = size * num_bytes; + auto* mem = static_cast(userp); + + char* ptr = static_cast(realloc(mem->data, mem->size + real_size + 1)); + if (ptr == nullptr) return 0; + + mem->data = ptr; + auto data = static_cast(mem->data); + memcpy(&(data[mem->size]), contents, real_size); + mem->size += real_size; + data[mem->size] = 0; + + return real_size; +} diff --git a/oss-internship-2020/curl/callbacks/callbacks.h b/oss-internship-2020/curl/callbacks/callbacks.h new file mode 100644 index 0000000..306e2eb --- /dev/null +++ b/oss-internship-2020/curl/callbacks/callbacks.h @@ -0,0 +1,24 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TESTS_CALLBACKS_H +#define TESTS_CALLBACKS_H + +#include + +// Append contents to the string stored by userp, which is a sapi::LenValStruct* +extern "C" size_t WriteToMemory(char* contents, size_t size, size_t num_bytes, + void* userp); + +#endif // TESTS_CALLBACKS_H diff --git a/oss-internship-2020/curl/curl_wrapper/CMakeLists.txt b/oss-internship-2020/curl/curl_wrapper/CMakeLists.txt new file mode 100644 index 0000000..78d2526 --- /dev/null +++ b/oss-internship-2020/curl/curl_wrapper/CMakeLists.txt @@ -0,0 +1,37 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wrapper library including curl, wrappers for some methods and callbacks +# The CURL_SAPI_CALLBACKS variable should contain the absolute paths of +# all the files implementing the callbacks +add_library(curl_wrapper_and_callbacks OBJECT + curl_wrapper.h + curl_wrapper.cc + "${CURL_SAPI_CALLBACKS}" +) +set_target_properties(curl_wrapper_and_callbacks + PROPERTIES LINKER_LANGUAGE C +) + +# Flags needed to build curl statically +set(CURL_HIDDEN_SYMBOLS OFF) +set(BUILD_SHARED_LIBS OFF) + +# Link the wrapper to the original curl library (testing is disabled in curl) +set(BUILD_TESTING OFF) +add_subdirectory(curl) +target_link_libraries(curl_wrapper_and_callbacks + CURL::libcurl + sapi::sapi +) diff --git a/oss-internship-2020/curl/curl_wrapper/curl_wrapper.cc b/oss-internship-2020/curl/curl_wrapper/curl_wrapper.cc new file mode 100644 index 0000000..6105fbe --- /dev/null +++ b/oss-internship-2020/curl/curl_wrapper/curl_wrapper.cc @@ -0,0 +1,86 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "curl_wrapper.h" // NOLINT(build/include) + +CURLcode curl_easy_setopt_ptr(CURL* handle, CURLoption option, + void* parameter) { + return curl_easy_setopt(handle, option, parameter); +} + +CURLcode curl_easy_setopt_long(CURL* handle, CURLoption option, + long parameter) { + return curl_easy_setopt(handle, option, parameter); +} + +CURLcode curl_easy_setopt_curl_off_t(CURL* handle, CURLoption option, + curl_off_t parameter) { + return curl_easy_setopt(handle, option, parameter); +} + +CURLcode curl_easy_getinfo_ptr(CURL* handle, CURLINFO option, void* parameter) { + return curl_easy_getinfo(handle, option, parameter); +} + +time_t_sapi curl_getdate_sapi(char* datestring, time_t_sapi* now) { + return curl_getdate(datestring, now); +} + +CURLMcode curl_multi_fdset_sapi(CURLM* multi_handle, fd_set_sapi* read_fd_set, + fd_set_sapi* write_fd_set, + fd_set_sapi* exc_fd_set, int* max_fd) { + return curl_multi_fdset(multi_handle, read_fd_set, write_fd_set, exc_fd_set, + max_fd); +} + +CURLMcode curl_multi_setopt_ptr(CURLM* handle, CURLMoption option, + void* parameter) { + return curl_multi_setopt(handle, option, parameter); +} + +CURLMcode curl_multi_setopt_long(CURLM* handle, CURLMoption option, + long parameter) { + return curl_multi_setopt(handle, option, parameter); +} + +CURLMcode curl_multi_setopt_curl_off_t(CURLM* handle, CURLMoption option, + curl_off_t parameter) { + return curl_multi_setopt(handle, option, parameter); +} + +CURLMcode curl_multi_poll_sapi(CURLM* multi_handle, + struct curl_waitfd* extra_fds, + unsigned int extra_nfds, int timeout_ms, + int* numfds) { + return curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms, + numfds); +} + +CURLMcode curl_multi_wait_sapi(CURLM* multi_handle, + struct curl_waitfd* extra_fds, + unsigned int extra_nfds, int timeout_ms, + int* numfds) { + return curl_multi_wait(multi_handle, extra_fds, extra_nfds, timeout_ms, + numfds); +} + +CURLSHcode curl_share_setopt_ptr(CURLSH* handle, CURLSHoption option, + void* parameter) { + return curl_share_setopt(handle, option, parameter); +} + +CURLSHcode curl_share_setopt_long(CURLSH* handle, CURLSHoption option, + long parameter) { + return curl_share_setopt(handle, option, parameter); +} diff --git a/oss-internship-2020/curl/curl_wrapper/curl_wrapper.h b/oss-internship-2020/curl/curl_wrapper/curl_wrapper.h new file mode 100644 index 0000000..4ef612b --- /dev/null +++ b/oss-internship-2020/curl/curl_wrapper/curl_wrapper.h @@ -0,0 +1,82 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Wrapper for curl library + +#ifndef CURL_WRAPPER_H +#define CURL_WRAPPER_H + +#include + +extern "C" { + +// The wrapper method is needed to make the variadic argument explicit +CURLcode curl_easy_setopt_ptr(CURL* handle, CURLoption option, void* parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLcode curl_easy_setopt_long(CURL* handle, CURLoption option, long parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLcode curl_easy_setopt_curl_off_t(CURL* handle, CURLoption option, + curl_off_t parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLcode curl_easy_getinfo_ptr(CURL* handle, CURLINFO option, void* parameter); + +// The typedef and wrapper method are needed because the original method has +// some conflicts in curl_sapi.sapi.h +typedef time_t time_t_sapi; +time_t_sapi curl_getdate_sapi(char* datestring, time_t_sapi* now); + +// The typedef and wrapper method are needed because the original method has +// some conflicts in curl_sapi.sapi.h +typedef fd_set fd_set_sapi; +CURLMcode curl_multi_fdset_sapi(CURLM* multi_handle, fd_set_sapi* read_fd_set, + fd_set_sapi* write_fd_set, + fd_set_sapi* exc_fd_set, int* max_fd); + +// The wrapper method is needed to make the variadic argument explicit +CURLMcode curl_multi_setopt_ptr(CURLM* handle, CURLMoption option, + void* parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLMcode curl_multi_setopt_long(CURLM* handle, CURLMoption option, + long parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLMcode curl_multi_setopt_curl_off_t(CURLM* handle, CURLMoption option, + curl_off_t parameter); + +// The wrapper method is needed because incomplete array type is not supported +CURLMcode curl_multi_poll_sapi(CURLM* multi_handle, + struct curl_waitfd* extra_fds, + unsigned int extra_nfds, int timeout_ms, + int* numfds); + +// The wrapper method is needed because incomplete array type is not supported +CURLMcode curl_multi_wait_sapi(CURLM* multi_handle, + struct curl_waitfd* extra_fds, + unsigned int extra_nfds, int timeout_ms, + int* numfds); + +// The wrapper method is needed to make the variadic argument explicit +CURLSHcode curl_share_setopt_ptr(CURLSH* handle, CURLSHoption option, + void* parameter); + +// The wrapper method is needed to make the variadic argument explicit +CURLSHcode curl_share_setopt_long(CURLSH* handle, CURLSHoption option, + long parameter); +} + +#endif // CURL_WRAPPER_H diff --git a/oss-internship-2020/curl/examples/CMakeLists.txt b/oss-internship-2020/curl/examples/CMakeLists.txt new file mode 100644 index 0000000..9193109 --- /dev/null +++ b/oss-internship-2020/curl/examples/CMakeLists.txt @@ -0,0 +1,76 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# All the examples are sandboxed versions of curl's examples +# (https://curl.haxx.se/libcurl/c/example.html) + +# Example 1: simple.c +add_executable(example1 + example1.cc + ../sandbox.h +) +target_link_libraries(example1 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 2: getinmemory.c +add_executable(example2 + example2.cc + ../sandbox.h +) +target_link_libraries(example2 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 3: simplessl.c +add_executable(example3 + example3.cc + ../sandbox.h +) +target_link_libraries(example3 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 4: multi-poll.c +add_executable(example4 + example4.cc + ../sandbox.h +) +target_link_libraries(example4 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 5: multithread.c +add_executable(example5 + example5.cc + ../sandbox.h +) +target_link_libraries(example5 PRIVATE + curl_sapi + sapi::sapi +) + +# Example 6: simple.c (using transactions) +add_executable(example6 + example6.cc + ../sandbox.h +) +target_link_libraries(example6 PRIVATE + curl_sapi + sapi::sapi +) diff --git a/oss-internship-2020/curl/examples/README.md b/oss-internship-2020/curl/examples/README.md new file mode 100644 index 0000000..d653a47 --- /dev/null +++ b/oss-internship-2020/curl/examples/README.md @@ -0,0 +1,38 @@ +# LibCurl Sandbox Examples + +Each example in this folder is the sandboxed version of a code snippet from +[this page](https://curl.haxx.se/libcurl/c/example.html) on curl's website. +These examples perform some basic tasks using libcurl, and can be useful both to +understand how to use LibCurl Sandbox, but also to get an idea of how regular +and sandboxed code compare to each other. + +This is the list of the examples: + +- **example1**: sandboxed version of + [simple.c](https://curl.haxx.se/libcurl/c/simple.html). Really simple HTTP + request, downloads and prints out the page at + [example.com](http://example.com). +- **example2**: sandboxed version of + [getinmemory.c](https://curl.haxx.se/libcurl/c/getinmemory.html). Same HTTP + request as example1. The difference is that this example uses a callback to + save the page directly in memory. Only the page size is printed out. +- **example3**: sandboxed version of + [simplessl.c](https://curl.haxx.se/libcurl/c/simplessl.html). HTTPS request + of the [example.com](https://example.com) page, using SSL authentication. + This script takes 4 arguments (SSL certificates file, SSL keys file, SSL + keys password and CA certificates files), and prints out the page. +- **example4**: sandboxed version of + [multi-poll.c](https://curl.haxx.se/libcurl/c/multi-poll.html). Same HTTP + request as example1, with the addition of a polling method that can be used + to track the status of the request. The page is printed out after it is + downloaded. +- **example5**: sandboxed version of + [multithread.c](https://curl.haxx.se/libcurl/c/multithread.html). Four HTTP + request of the pages [example.com](http://example.com), + [example.edu](http://example.edu), [example.net](http://example.net) and + [example.org](http://example.org), performed at the same time using + libcurl's multithreading methods. The pages are printed out. +- **example6**: sandboxed version of + [simple.c](https://curl.haxx.se/libcurl/c/simple.html). Performs the same + tasks as example1, but Sandbox API Transactions are used to show how they + can be used to perform a simple request. diff --git a/oss-internship-2020/curl/examples/example1.cc b/oss-internship-2020/curl/examples/example1.cc new file mode 100644 index 0000000..e5ee5fe --- /dev/null +++ b/oss-internship-2020/curl/examples/example1.cc @@ -0,0 +1,78 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of simple.c +// Simple HTTP GET request + +#include + +#include "../sandbox.h" // NOLINT(build/include) + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Initialize sandbox2 and sapi + CurlSapiSandbox sandbox; + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + // Initialize the curl session + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr curl(curl_handle.value()); + if (!curl.GetValue()) LOG(FATAL) << "curl_easy_init failed: curl is NULL"; + + absl::StatusOr curl_code; + + // Specify URL to get + sapi::v::ConstCStr url("http://example.com"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the library to follow a redirection + curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_FOLLOWLOCATION, 1l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status(); + } + + // Disable authentication of peer certificate + curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_SSL_VERIFYPEER, 0l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status(); + } + + // Perform the request + curl_code = api.curl_easy_perform(&curl); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status(); + } + + // Cleanup curl + status = api.curl_easy_cleanup(&curl); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example2.cc b/oss-internship-2020/curl/examples/example2.cc new file mode 100644 index 0000000..c7abe0c --- /dev/null +++ b/oss-internship-2020/curl/examples/example2.cc @@ -0,0 +1,107 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of getinmemory.c +// HTTP GET request using callbacks + +#include +#include + +#include "../sandbox.h" // NOLINT(build/include) + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Initialize sandbox2 and sapi + CurlSapiSandbox sandbox; + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + // Generate pointer to WriteMemoryCallback function + void* function_ptr; + status = sandbox.rpc_channel()->Symbol("WriteToMemory", &function_ptr); + if (!status.ok()) { + LOG(FATAL) << "sapi::Sandbox::rpc_channel().Symbol failed: " << status; + } + sapi::v::RemotePtr remote_function_ptr(function_ptr); + + // Initialize the curl session + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr curl(curl_handle.value()); + if (!curl.GetValue()) { + LOG(FATAL) << "curl_easy_init failed: curl is NULL"; + } + + absl::StatusOr curl_code; + + // Specify URL to get + sapi::v::ConstCStr url("http://example.com"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set WriteMemoryCallback as the write function + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_WRITEFUNCTION, + &remote_function_ptr); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Pass 'chunk' struct to the callback function + sapi::v::LenVal chunk(0); + curl_code = + api.curl_easy_setopt_ptr(&curl, CURLOPT_WRITEDATA, chunk.PtrBoth()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set a user agent + sapi::v::ConstCStr user_agent("libcurl-agent/1.0"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_USERAGENT, + user_agent.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Perform the request + curl_code = api.curl_easy_perform(&curl); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status(); + } + + // Retrieve memory size + status = sandbox.TransferFromSandboxee(&chunk); + if (!status.ok()) { + LOG(FATAL) << "sandbox.TransferFromSandboxee failed: " << status; + } + std::cout << "memory size: " << chunk.GetDataSize() << " bytes" << std::endl; + + // Cleanup curl + status = api.curl_easy_cleanup(&curl); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example3.cc b/oss-internship-2020/curl/examples/example3.cc new file mode 100644 index 0000000..1a73c93 --- /dev/null +++ b/oss-internship-2020/curl/examples/example3.cc @@ -0,0 +1,166 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of simplessl.c +// HTTPS GET request + +#include + +#include "../sandbox.h" // NOLINT(build/include) + +class CurlSapiSandboxEx3 : public CurlSapiSandbox { + public: + CurlSapiSandboxEx3(std::string ssl_certificate, std::string ssl_key, + std::string ca_certificates) + : ssl_certificate(ssl_certificate), + ssl_key(ssl_key), + ca_certificates(ca_certificates) {} + + private: + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder*) override { + // Add the syscalls and files missing in CurlSandbox to a new PolicyBuilder + auto policy_builder = std::make_unique(); + (*policy_builder) + .AllowFutexOp(FUTEX_WAIT_PRIVATE) + .AllowGetPIDs() + .AllowGetRandom() + .AllowHandleSignals() + .AllowSyscall(__NR_sysinfo) + .AddFile(ssl_certificate) + .AddFile(ssl_key) + .AddFile(ca_certificates); + // Provide the new PolicyBuilder to ModifyPolicy in CurlSandbox + return CurlSapiSandbox::ModifyPolicy(policy_builder.get()); + } + + std::string ssl_certificate; + std::string ssl_key; + std::string ca_certificates; +}; + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Get input parameters (should be absolute paths) + if (argc != 5) { + LOG(FATAL) << "wrong number of arguments (4 expected)"; + } + std::string ssl_certificate = argv[1]; + std::string ssl_key = argv[2]; + std::string ssl_key_password = argv[3]; + std::string ca_certificates = argv[4]; + + // Initialize sandbox2 and sapi + CurlSapiSandboxEx3 sandbox(ssl_certificate, ssl_key, ca_certificates); + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + absl::StatusOr curl_code; + + // Initialize curl (CURL_GLOBAL_DEFAULT = 3) + curl_code = api.curl_global_init(3l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_global_init failed: " << curl_code.status(); + } + + // Initialize curl easy handle + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr curl(curl_handle.value()); + if (!curl.GetValue()) { + LOG(FATAL) << "curl_easy_init failed: curl is NULL"; + } + + // Specify URL to get (using HTTPS) + sapi::v::ConstCStr url("https://example.com"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the SSL certificate type to "PEM" + sapi::v::ConstCStr ssl_cert_type("PEM"); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLCERTTYPE, + ssl_cert_type.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the certificate for client authentication + sapi::v::ConstCStr sapi_ssl_certificate(ssl_certificate.c_str()); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLCERT, + sapi_ssl_certificate.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the private key for client authentication + sapi::v::ConstCStr sapi_ssl_key(ssl_key.c_str()); + curl_code = + api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLKEY, sapi_ssl_key.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the password used to protect the private key + sapi::v::ConstCStr sapi_ssl_key_password(ssl_key_password.c_str()); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_KEYPASSWD, + sapi_ssl_key_password.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Set the file with the certificates vaildating the server + sapi::v::ConstCStr sapi_ca_certificates(ca_certificates.c_str()); + curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_CAINFO, + sapi_ca_certificates.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Verify the authenticity of the server + curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_SSL_VERIFYPEER, 1L); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status(); + } + + // Perform the request + curl_code = api.curl_easy_perform(&curl); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status(); + } + + // Cleanup curl easy handle + status = api.curl_easy_cleanup(&curl); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } + + // Cleanup curl + status = api.curl_global_cleanup(); + if (!status.ok()) { + LOG(FATAL) << "curl_global_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example4.cc b/oss-internship-2020/curl/examples/example4.cc new file mode 100644 index 0000000..9103866 --- /dev/null +++ b/oss-internship-2020/curl/examples/example4.cc @@ -0,0 +1,128 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of multi-poll.c +// HTTP GET request with polling + +#include + +#include "../sandbox.h" // NOLINT(build/include) +#include "curl_sapi.sapi.h" // NOLINT(build/include) +#include "sandboxed_api/util/flag.h" + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Initialize sandbox2 and sapi + CurlSapiSandbox sandbox; + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + // Number of running handles + sapi::v::Int still_running(1); + + absl::StatusOr curl_code; + + // Initialize curl (CURL_GLOBAL_DEFAULT = 3) + curl_code = api.curl_global_init(3l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_global_init failed: " << curl_code.status(); + } + + // Initialize http_handle + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr http_handle(curl_handle.value()); + if (!http_handle.GetValue()) { + LOG(FATAL) << "curl_easy_init failed: http_handle is NULL"; + } + + // Specify URL to get + sapi::v::ConstCStr url("http://example.com"); + curl_code = + api.curl_easy_setopt_ptr(&http_handle, CURLOPT_URL, url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Initialize multi_handle + absl::StatusOr curlm_handle = api.curl_multi_init(); + if (!curlm_handle.ok()) { + LOG(FATAL) << "curl_multi_init failed: " << curlm_handle.status(); + } + sapi::v::RemotePtr multi_handle(curlm_handle.value()); + if (!multi_handle.GetValue()) { + LOG(FATAL) << "curl_multi_init failed: multi_handle is NULL"; + } + + // Add http_handle to the multi stack + curl_code = api.curl_multi_add_handle(&multi_handle, &http_handle); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_multi_add_handle failed: " << curl_code.status(); + } + + while (still_running.GetValue()) { + sapi::v::Int numfds(0); + + // Perform the request + curl_code = api.curl_multi_perform(&multi_handle, still_running.PtrBoth()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_mutli_perform failed: " << curl_code.status(); + } + + if (still_running.GetValue()) { + // Wait for an event or timeout + sapi::v::NullPtr null_ptr; + curl_code = api.curl_multi_poll_sapi(&multi_handle, &null_ptr, 0, 1000, + numfds.PtrBoth()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_multi_poll_sapi failed: " << curl_code.status(); + } + } + } + + // Remove http_handle from the multi stack + curl_code = api.curl_multi_remove_handle(&multi_handle, &http_handle); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_multi_remove_handle failed: " << curl_code.status(); + } + + // Cleanup http_handle + status = api.curl_easy_cleanup(&http_handle); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } + + // Cleanup multi_handle + curl_code = api.curl_multi_cleanup(&multi_handle); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_multi_cleanup failed: " << curl_code.status(); + } + + // Cleanup curl + status = api.curl_global_cleanup(); + if (!status.ok()) { + LOG(FATAL) << "curl_global_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example5.cc b/oss-internship-2020/curl/examples/example5.cc new file mode 100644 index 0000000..8cc6a16 --- /dev/null +++ b/oss-internship-2020/curl/examples/example5.cc @@ -0,0 +1,102 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of multithread.c +// Multithreaded HTTP GET requests + +#include + +#include + +#include "../sandbox.h" // NOLINT(build/include) + +void pull_one_url(const std::string& url, CurlApi& api) { + // Initialize the curl session + absl::StatusOr curl_handle = api.curl_easy_init(); + if (!curl_handle.ok()) { + LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status(); + } + sapi::v::RemotePtr curl(curl_handle.value()); + if (!curl.GetValue()) { + LOG(FATAL) << "curl_easy_init failed: curl is NULL"; + } + + absl::StatusOr curl_code; + + // Specify URL to get + sapi::v::ConstCStr sapi_url(url.c_str()); + curl_code = + api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, sapi_url.PtrBefore()); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status(); + } + + // Perform the request + curl_code = api.curl_easy_perform(&curl); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status(); + } + + // Cleanup curl + absl::Status status = api.curl_easy_cleanup(&curl); + if (!status.ok()) { + LOG(FATAL) << "curl_easy_cleanup failed: " << status; + } +} + +const std::vector urls = { + "http://example.com", "http://example.edu", "http://example.net", + "http://example.org"}; + +int main(int argc, char* argv[]) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); + + absl::Status status; + + // Initialize sandbox2 and sapi + CurlSapiSandbox sandbox; + status = sandbox.Init(); + if (!status.ok()) { + LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status; + } + CurlApi api(&sandbox); + + absl::StatusOr curl_code; + + // Initialize curl (CURL_GLOBAL_DEFAULT = 3) + curl_code = api.curl_global_init(3l); + if (!curl_code.ok() || curl_code.value() != CURLE_OK) { + LOG(FATAL) << "curl_global_init failed: " << curl_code.status(); + } + + // Create the threads + std::vector threads; + for (auto& url : urls) { + threads.emplace_back(pull_one_url, std::ref(url), std::ref(api)); + } + + // Join the threads + for (auto& thread : threads) { + thread.join(); + } + + // Cleanup curl + status = api.curl_global_cleanup(); + if (!status.ok()) { + LOG(FATAL) << "curl_global_cleanup failed: " << status; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/examples/example6.cc b/oss-internship-2020/curl/examples/example6.cc new file mode 100644 index 0000000..15fb1c2 --- /dev/null +++ b/oss-internship-2020/curl/examples/example6.cc @@ -0,0 +1,72 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sandboxed version of simple.c using transactions +// Simple HTTP GET request + +#include + +#include "../sandbox.h" // NOLINT(build/include) +#include "sandboxed_api/transaction.h" + +class CurlTransaction : public sapi::Transaction { + public: + explicit CurlTransaction(std::unique_ptr sandbox) + : sapi::Transaction(std::move(sandbox)) { + sapi::Transaction::SetTimeLimit(kTimeOutVal); + } + + private: + // Default timeout value for each transaction run. + static constexpr time_t kTimeOutVal = 2; + + // The main processing function. + absl::Status Main() override; +}; + +absl::Status CurlTransaction::Main() { + CurlApi api(sandbox()); + + // Initialize the curl session + SAPI_ASSIGN_OR_RETURN(void* curl_remote, api.curl_easy_init()); + sapi::v::RemotePtr curl(curl_remote); + TRANSACTION_FAIL_IF_NOT(curl.GetValue(), "curl_easy_init failed"); + + // Specify URL to get + sapi::v::ConstCStr url("http://example.com"); + SAPI_ASSIGN_OR_RETURN( + int setopt_url_code, + api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore())); + TRANSACTION_FAIL_IF_NOT(setopt_url_code == CURLE_OK, + "curl_easy_setopt_ptr failed"); + + // Perform the request + SAPI_ASSIGN_OR_RETURN(int perform_code, api.curl_easy_perform(&curl)); + TRANSACTION_FAIL_IF_NOT(setopt_url_code == CURLE_OK, + "curl_easy_perform failed"); + + // Cleanup curl + TRANSACTION_FAIL_IF_NOT(api.curl_easy_cleanup(&curl).ok(), + "curl_easy_cleanup failed"); + + return absl::OkStatus(); +} + +int main(int argc, char* argv[]) { + CurlTransaction curl{std::make_unique()}; + absl::Status status = curl.Run(); + CHECK(status.ok()) << "CurlTransaction failed"; + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/curl/sandbox.h b/oss-internship-2020/curl/sandbox.h new file mode 100644 index 0000000..914617f --- /dev/null +++ b/oss-internship-2020/curl/sandbox.h @@ -0,0 +1,67 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SANDBOX_H_ +#define SANDBOX_H_ + +#include +#include // For mmap arguments +#include + +#include + +#include "curl_sapi.sapi.h" // NOLINT(build/include) +#include "sandboxed_api/sandbox2/util/bpf_helper.h" + +class CurlSapiSandbox : public CurlSandbox { + protected: + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder* policy_builder) override { + // Return a new policy + return (*policy_builder) + .AllowDynamicStartup() + .AllowExit() + .AllowFork() + .AllowFutexOp(FUTEX_WAKE_PRIVATE) + .AllowMmap() + .AllowOpen() + .AllowRead() + .AllowSafeFcntl() + .AllowWrite() + .AllowSyscalls({ + __NR_accept, + __NR_access, + __NR_bind, + __NR_connect, + __NR_getpeername, + __NR_getsockname, + __NR_getsockopt, + __NR_ioctl, + __NR_listen, + __NR_madvise, + __NR_poll, + __NR_recvfrom, + __NR_recvmsg, + __NR_sendmmsg, + __NR_sendto, + __NR_setsockopt, + __NR_socket, + }) + .AllowUnrestrictedNetworking() + .AddDirectory("/lib") + .BuildOrDie(); + } +}; + +#endif // SANDBOX_H_ diff --git a/oss-internship-2020/curl/tests/CMakeLists.txt b/oss-internship-2020/curl/tests/CMakeLists.txt new file mode 100644 index 0000000..6fd4556 --- /dev/null +++ b/oss-internship-2020/curl/tests/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(GoogleTest) + +add_executable(tests + test_utils.h + test_utils.cc + tests.cc +) + +target_link_libraries(tests + curl_sapi sapi::sapi + gtest gmock gtest_main +) + +gtest_discover_tests(tests) diff --git a/oss-internship-2020/curl/tests/test_utils.cc b/oss-internship-2020/curl/tests/test_utils.cc new file mode 100644 index 0000000..8ea2897 --- /dev/null +++ b/oss-internship-2020/curl/tests/test_utils.cc @@ -0,0 +1,301 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "test_utils.h" // NOLINT(build/include) + +#include +#include +#include +#include +#include +#include +#include + +#include +#include // NOLINT(build/c++11) + +int CurlTestUtils::port_; +std::thread CurlTestUtils::server_thread_; + +absl::Status CurlTestUtils::CurlTestSetUp() { + // Initialize sandbox2 and sapi + sandbox_ = std::make_unique(); + absl::Status init = sandbox_->Init(); + if (!init.ok()) { + return init; + } + api_ = std::make_unique(sandbox_.get()); + + // Initialize curl + absl::StatusOr curl_handle = api_->curl_easy_init(); + if (!curl_handle.ok()) { + return curl_handle.status(); + } + if (!curl_handle.value()) { + return absl::UnavailableError("curl_easy_init returned NULL "); + } + curl_ = std::make_unique(curl_handle.value()); + + absl::StatusOr curl_code; + + // Specify request URL + sapi::v::ConstCStr sapi_url(kUrl.data()); + curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_URL, + sapi_url.PtrBefore()); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_setopt_ptr returned with the error code " + + curl_code.value()); + } + + // Set port + curl_code = api_->curl_easy_setopt_long(curl_.get(), CURLOPT_PORT, port_); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_setopt_long returned with the error code " + + curl_code.value()); + } + + // Generate pointer to the WriteToMemory callback + void* function_ptr; + absl::Status symbol = + sandbox_->rpc_channel()->Symbol("WriteToMemory", &function_ptr); + if (!symbol.ok()) { + return symbol; + } + sapi::v::RemotePtr remote_function_ptr(function_ptr); + + // Set WriteToMemory as the write function + curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_WRITEFUNCTION, + &remote_function_ptr); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_setopt_ptr returned with the error code " + + curl_code.value()); + } + + // Pass memory chunk object to the callback + chunk_ = std::make_unique(0); + curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_WRITEDATA, + chunk_->PtrBoth()); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_setopt_ptr returned with the error code " + + curl_code.value()); + } + + return absl::OkStatus(); +} + +absl::Status CurlTestUtils::CurlTestTearDown() { + // Cleanup curl + return api_->curl_easy_cleanup(curl_.get()); +} + +absl::StatusOr CurlTestUtils::PerformRequest() { + // Perform the request + absl::StatusOr curl_code = api_->curl_easy_perform(curl_.get()); + if (!curl_code.ok()) { + return curl_code.status(); + } + if (curl_code.value() != CURLE_OK) { + return absl::UnavailableError( + "curl_easy_perform returned with the error code " + curl_code.value()); + } + + // Get pointer to the memory chunk + absl::Status status = sandbox_->TransferFromSandboxee(chunk_.get()); + if (!status.ok()) { + return status; + } + + return std::string{reinterpret_cast(chunk_->GetData())}; +} + +namespace { + +// Read the socket until str is completely read +std::string ReadUntil(const int socket, const std::string& str, + const size_t max_request_size) { + std::string str_read; + str_read.reserve(max_request_size); + + // Read one char at a time until str is suffix of buf + while (!absl::EndsWith(str_read, str)) { + char next_char; + if (str_read.size() >= max_request_size || + read(socket, &next_char, 1) < 1) { + return ""; + } + str_read += next_char; + } + + return str_read; +} + +// Parse HTTP headers to return the Content-Length +ssize_t GetContentLength(const std::string& headers) { + // Find the Content-Length header + std::string::size_type length_header_start = headers.find("Content-Length: "); + + // There is no Content-Length field + if (length_header_start == std::string::npos) { + return 0; + } + + // Find Content-Length string + std::string::size_type length_start = + length_header_start + std::string{"Content-Length: "}.size(); + std::string::size_type length_bytes = + headers.find("\r\n", length_start) - length_start; + + // length_bytes exceeds maximum + if (length_bytes >= 64) { + return -1; + } + + // Convert string to int and return + return std::stoi(headers.substr(length_start, length_bytes)); +} + +// Read exactly content_bytes from the socket +std::string ReadExact(int socket, size_t content_bytes) { + std::string str_read; + str_read.reserve(content_bytes); + + // Read one char at a time until all chars are read + while (str_read.size() < content_bytes) { + char next_char; + if (read(socket, &next_char, 1) < 1) { + return ""; + } + str_read += next_char; + } + + return str_read; +} + +// Listen on the socket and answer back to requests +void ServerLoop(int listening_socket, sockaddr_in socket_address) { + socklen_t socket_address_size = sizeof(socket_address); + + // Listen on the socket (maximum 1 connection) + if (listen(listening_socket, 1) == -1) { + return; + } + + // Keep accepting connections until the thread is terminated + // (i.e. server_thread_ is assigned to a new thread or destroyed) + for (;;) { + // File descriptor to the connection socket + // This blocks the thread until a connection is established + int accepted_socket = + accept(listening_socket, reinterpret_cast(&socket_address), + reinterpret_cast(&socket_address_size)); + if (accepted_socket == -1) { + return; + } + + constexpr int kMaxRequestSize = 4096; + + // Read until the end of the headers + std::string headers = + ReadUntil(accepted_socket, "\r\n\r\n", kMaxRequestSize); + + if (headers == "") { + close(accepted_socket); + return; + } + + // Get the length of the request content + ssize_t content_length = GetContentLength(headers); + if (content_length > kMaxRequestSize - headers.size() || + content_length < 0) { + close(accepted_socket); + return; + } + + // Read the request content + std::string content = ReadExact(accepted_socket, content_length); + + // Prepare a response for the request + std::string http_response = + "HTTP/1.1 200 OK\nContent-Type: text/plain\nContent-Length: "; + + if (headers.substr(0, 3) == "GET") { + http_response += "2\r\n\r\nOK"; + + } else if (headers.substr(0, 4) == "POST") { + http_response += + std::to_string(content.size()) + "\r\n\r\n" + std::string{content}; + + } else { + close(accepted_socket); + return; + } + + // Ignore any errors, the connection will be closed anyway + write(accepted_socket, http_response.c_str(), http_response.size()); + + // Close the socket + close(accepted_socket); + } +} + +} // namespace + +void CurlTestUtils::StartMockServer() { + // Get the socket file descriptor + int listening_socket = socket(AF_INET, SOCK_STREAM, 0); + + // Create the socket address object + // The port is set to 0, meaning that it will be auto assigned + // Only local connections can access this socket + sockaddr_in socket_address{AF_INET, 0, htonl(INADDR_LOOPBACK)}; + socklen_t socket_address_size = sizeof(socket_address); + if (listening_socket == -1) { + return; + } + + // Bind the file descriptor to the socket address object + if (bind(listening_socket, reinterpret_cast(&socket_address), + socket_address_size) == -1) { + return; + } + + // Assign an available port to the socket address object + if (getsockname(listening_socket, + reinterpret_cast(&socket_address), + &socket_address_size) == -1) { + return; + } + + // Get the port number + port_ = ntohs(socket_address.sin_port); + + // Set server_thread_ operation to socket listening + server_thread_ = std::thread(ServerLoop, listening_socket, socket_address); +} diff --git a/oss-internship-2020/curl/tests/test_utils.h b/oss-internship-2020/curl/tests/test_utils.h new file mode 100644 index 0000000..fa74769 --- /dev/null +++ b/oss-internship-2020/curl/tests/test_utils.h @@ -0,0 +1,55 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TESTS_H_ +#define TESTS_H_ + +#include "../sandbox.h" // NOLINT(build/include) +#include "curl_sapi.sapi.h" // NOLINT(build/include) +#include "gtest/gtest.h" +#include "sandboxed_api/util/flag.h" +#include "sandboxed_api/util/status_matchers.h" + +// Helper class that can be used to test Curl Sandboxed +class CurlTestUtils { + protected: + // Initialize and set up the curl handle + absl::Status CurlTestSetUp(); + // Clean up the curl handle + absl::Status CurlTestTearDown(); + + // Perform a request to the mock server, return the response + absl::StatusOr PerformRequest(); + + // Start a mock server (only once) that will manage connections for the tests + // The server listens on a port asynchronously by creating a thread + // The port number is stored in port_ + // Responds with "OK" to a GET request + // Responds with the POST request fields to a POST request + static void StartMockServer(); + + std::unique_ptr sandbox_; + std::unique_ptr api_; + std::unique_ptr curl_; + + static std::thread server_thread_; + + static constexpr absl::string_view kUrl = "http://127.0.0.1/"; + static int port_; + + private: + std::unique_ptr chunk_; +}; + +#endif // TESTS_H_ diff --git a/oss-internship-2020/curl/tests/tests.cc b/oss-internship-2020/curl/tests/tests.cc new file mode 100644 index 0000000..c786169 --- /dev/null +++ b/oss-internship-2020/curl/tests/tests.cc @@ -0,0 +1,138 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "test_utils.h" // NOLINT(build/include) + +class CurlTest : public CurlTestUtils, public ::testing::Test { + protected: + void SetUp() override { + // Start mock server, get port number and check for any error + StartMockServer(); + ASSERT_TRUE(server_thread_.joinable()); + ASSERT_TRUE(CurlTestSetUp().ok()); + } + + void TearDown() override { + ASSERT_TRUE(CurlTestTearDown().ok()); + // Detach the server thread + server_thread_.detach(); + } +}; + +TEST_F(CurlTest, EffectiveUrl) { + sapi::v::RemotePtr effective_url_ptr(nullptr); + + ASSERT_TRUE(PerformRequest().ok()); + + // Get effective URL + SAPI_ASSERT_OK_AND_ASSIGN( + int getinfo_code, + api_->curl_easy_getinfo_ptr(curl_.get(), CURLINFO_EFFECTIVE_URL, + effective_url_ptr.PtrBoth())); + ASSERT_EQ(getinfo_code, CURLE_OK); + + // Store effective URL in a string + SAPI_ASSERT_OK_AND_ASSIGN(std::string effective_url, + sandbox_->GetCString(sapi::v::RemotePtr( + effective_url_ptr.GetPointedVar()))); + + // Compare effective URL with original URL + ASSERT_EQ(effective_url, kUrl); +} + +TEST_F(CurlTest, EffectivePort) { + sapi::v::Int effective_port; + + ASSERT_TRUE(PerformRequest().ok()); + + // Get effective port + SAPI_ASSERT_OK_AND_ASSIGN(int getinfo_code, api_->curl_easy_getinfo_ptr( + curl_.get(), CURLINFO_PRIMARY_PORT, + effective_port.PtrBoth())); + ASSERT_EQ(getinfo_code, CURLE_OK); + + // Compare effective port with port set by the mock server + ASSERT_EQ(effective_port.GetValue(), port_); +} + +TEST_F(CurlTest, ResponseCode) { + sapi::v::Int response_code; + + ASSERT_TRUE(PerformRequest().ok()); + + // Get response code + SAPI_ASSERT_OK_AND_ASSIGN( + int getinfo_code, + api_->curl_easy_getinfo_ptr(curl_.get(), CURLINFO_RESPONSE_CODE, + response_code.PtrBoth())); + ASSERT_EQ(getinfo_code, CURLE_OK); + + // Check response code + ASSERT_EQ(response_code.GetValue(), 200); +} + +TEST_F(CurlTest, ContentType) { + sapi::v::RemotePtr content_type_ptr(nullptr); + + ASSERT_TRUE(PerformRequest().ok()); + + // Get effective URL + SAPI_ASSERT_OK_AND_ASSIGN(int getinfo_code, api_->curl_easy_getinfo_ptr( + curl_.get(), CURLINFO_CONTENT_TYPE, + content_type_ptr.PtrBoth())); + ASSERT_EQ(getinfo_code, CURLE_OK); + + // Store content type in a string + SAPI_ASSERT_OK_AND_ASSIGN(std::string content_type, + sandbox_->GetCString(sapi::v::RemotePtr( + content_type_ptr.GetPointedVar()))); + + // Compare content type with "text/plain" + ASSERT_EQ(content_type, "text/plain"); +} + +TEST_F(CurlTest, GETResponse) { + SAPI_ASSERT_OK_AND_ASSIGN(std::string response, PerformRequest()); + + // Compare response with expected response + ASSERT_EQ(response, "OK"); +} + +TEST_F(CurlTest, POSTResponse) { + sapi::v::ConstCStr post_fields("postfields"); + + // Set request method to POST + SAPI_ASSERT_OK_AND_ASSIGN(int setopt_post, api_->curl_easy_setopt_long( + curl_.get(), CURLOPT_POST, 1l)); + ASSERT_EQ(setopt_post, CURLE_OK); + + // Set the size of the POST fields + SAPI_ASSERT_OK_AND_ASSIGN( + int setopt_post_fields_size, + api_->curl_easy_setopt_long(curl_.get(), CURLOPT_POSTFIELDSIZE, + post_fields.GetSize())); + ASSERT_EQ(setopt_post_fields_size, CURLE_OK); + + // Set the POST fields + SAPI_ASSERT_OK_AND_ASSIGN( + int setopt_post_fields, + api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_POSTFIELDS, + post_fields.PtrBefore())); + ASSERT_EQ(setopt_post_fields, CURLE_OK); + + SAPI_ASSERT_OK_AND_ASSIGN(std::string response, PerformRequest()); + + // Compare response with expected response + ASSERT_EQ(std::string(post_fields.GetData()), response); +} diff --git a/oss-internship-2020/gdal/CANYrelief1-geo.tif b/oss-internship-2020/gdal/CANYrelief1-geo.tif new file mode 100644 index 0000000..f9d2ef0 Binary files /dev/null and b/oss-internship-2020/gdal/CANYrelief1-geo.tif differ diff --git a/oss-internship-2020/gdal/CMakeLists.txt b/oss-internship-2020/gdal/CMakeLists.txt new file mode 100644 index 0000000..708952f --- /dev/null +++ b/oss-internship-2020/gdal/CMakeLists.txt @@ -0,0 +1,86 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.10) + +project(test CXX C) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +find_package(PNG REQUIRED) +find_package(PkgConfig REQUIRED) +pkg_check_modules(libpcre REQUIRED IMPORTED_TARGET libpcre) +pkg_check_modules(proj REQUIRED IMPORTED_TARGET proj) + +set(SAPI_ROOT "${PROJECT_SOURCE_DIR}/../.." CACHE PATH "Path to the Sandboxed API source tree") +# cmake .. -G Ninja -DSAPI_ROOT=$HOME/sapi_root + +set(SAPI_ENABLE_EXAMPLES OFF CACHE BOOL "") +set(SAPI_ENABLE_TESTS OFF CACHE BOOL "") +add_subdirectory("${SAPI_ROOT}" + "${CMAKE_BINARY_DIR}/sandboxed-api-build" + # Omit this to have the full Sandboxed API in IDE + EXCLUDE_FROM_ALL) + +add_library(libgdal STATIC IMPORTED) +set_property(TARGET libgdal PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/lib/libgdal.a") + +target_link_libraries(libgdal INTERFACE + crypto + expat + jpeg + PkgConfig::libpcre + PkgConfig::proj + sqlite3 + tiff + z + pthread + m + rt + dl + curl + PNG::PNG +) + +add_sapi_library(gdal_sapi + FUNCTIONS GDALOpen + GDALAllRegister + GDALGetDatasetDriver + GDALGetDriverShortName + GDALGetDriverLongName + GDALGetGeoTransform + GDALGetRasterBand + GDALGetBlockSize + GDALGetRasterBandXSize + GDALGetRasterBandYSize + GDALRasterIO + INPUTS "/usr/include/gdal/gdal.h" + LIBRARY libgdal + LIBRARY_NAME GDAL + NAMESPACE "" +) + +target_include_directories(gdal_sapi INTERFACE + "${PROJECT_BINARY_DIR}" +) + +add_executable(raster + raster.cc +) + +target_link_libraries(raster + gdal_sapi + sapi::sapi +) diff --git a/oss-internship-2020/gdal/README.md b/oss-internship-2020/gdal/README.md new file mode 100644 index 0000000..d3713b7 --- /dev/null +++ b/oss-internship-2020/gdal/README.md @@ -0,0 +1,88 @@ +# GDAL Raster GeoTIFF Workflow + +``` +Build Tools: CMake/Ninja +OS: Linux +``` + +### For installing GDAL: + +``` +sudo apt-get install python3.6-dev +sudo add-apt-repository ppa:ubuntugis/ppa && sudo apt update +sudo apt-get install gdal-bin +sudo apt-get install libgdal-dev +``` + +### Dependencies: + +PNG: `sudo apt-get install libpng-dev` + +PCRE: `sudo apt-get install libpcre3 libpcre3-dev` + +PROJ: `sudo apt-get install libproj-dev` + +OBS! You may need to set `export LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib`. +It is required for libproj.so to be found into /usr/local/lib/. You can also fix +this by typing `locate libproj.so` which will give you + and then `cp +/usr/local/lib/`. + +### Initializing GDAL submodule: + +`git submodule add https://github.com/OSGeo/gdal/tree/master/gdal` + +### Building GDAL statically + +GNUmakefile from gdal/gdal can handle building the static library. + +`cd gdal/gdal && make static-lib` + +`cd ../.. && mkdir lib` + +`cp gdal/gdal/libgdal.a lib/` + +OBS! The file is huge! It may take a while. + +### For testing: + +`mkdir build && cd build` + +`cmake .. -G Ninja` + +`ninja` + +`./raster ` + +## About the project + +GDAL is a translator library for raster and vector geospatial data format. The +project consist in rastering a GeoTIFF file format using GDAL functionalities +and sandboxed methods. + +## Implementation + +*Sandboxing...* + +The purpose of sandboxing is to limit the permissions and capabilities of +library’s methods, in order to secure the usage of them. After obtaining the +sandbox, the functions will be called through an Sandbox API (being called api +in the current test) and so, the operations, system calls or namspaces access +may be controlled. + +*Raster process...* + +Useful functions from the `gdal.h` header are added to the SAPI library built +with CMake. + +One .tiff file is manipulated with GDALOpen functionality, which extracts a +pointer to the data set containg a list of raster bands, all pertaining to the +same area. Metadata, a coordinate system, a georeferencing transform, size of +raster and various other information are kept into the data set that corresponds +to the image. + +To create an array containing the image information, the dimentions needed are +extracted using some specific GDAL(X/Y)Size functions applied to the block. +GDALRasterBand function takes care of data type conversion, one more step +following: placing the converted data (with RasterIO method) into the created +and well allocated structure. diff --git a/oss-internship-2020/gdal/raster.cc b/oss-internship-2020/gdal/raster.cc new file mode 100644 index 0000000..f5865f8 --- /dev/null +++ b/oss-internship-2020/gdal/raster.cc @@ -0,0 +1,195 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include + +#include +#include "gdal_sapi.sapi.h" // NOLINT(build/include) +#include "sandboxed_api/sandbox2/util/fileops.h" + +class GdalSapiSandbox : public GDALSandbox { + public: + GdalSapiSandbox(std::string path) + : GDALSandbox(), file_path_(std::move(path)) {} + + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder*) override { + return sandbox2::PolicyBuilder() + .AllowDynamicStartup() + .AllowRead() + .AllowSystemMalloc() + .AllowWrite() + .AllowExit() + .AllowStat() + .AllowOpen() + .AllowSyscalls({ + __NR_futex, + __NR_close, + __NR_recvmsg, + __NR_getdents64, + __NR_lseek, + __NR_getpid, + __NR_sysinfo, + __NR_prlimit64, + __NR_ftruncate, + __NR_unlink, + }) + .AddFile(file_path_) + .BuildOrDie(); + } + + private: + std::string file_path_; +}; + +absl::Status GdalMain(std::string filename) { + // Reading GDALDataset from a (local, specific) file. + GdalSapiSandbox sandbox(filename); + + SAPI_RETURN_IF_ERROR(sandbox.Init()); + GDALApi api(&sandbox); + + sapi::v::CStr s(filename.data()); + + SAPI_RETURN_IF_ERROR(api.GDALAllRegister()); + auto open = api.GDALOpen(s.PtrBoth(), GDALAccess::GA_ReadOnly); + + LOG(INFO) << "Dataset pointer adress: " << open.value() << std::endl; + sapi::v::RemotePtr ptr_dataset(open.value()); + + LOG(INFO) << ptr_dataset.ToString() << std::endl; + if (!open.value()) { + return absl::AbortedError("NULL pointer for Dataset.\n"); + } + + // Printing some general information about the dataset. + auto driver = api.GDALGetDatasetDriver(&ptr_dataset); + sapi::v::RemotePtr ptr_driver(driver.value()); + + auto driver_short_name = api.GDALGetDriverShortName(&ptr_driver); + auto driver_long_name = api.GDALGetDriverLongName(&ptr_driver); + + sapi::v::RemotePtr ptr_driver_short_name(driver_short_name.value()); + sapi::v::RemotePtr ptr_driver_long_name(driver_long_name.value()); + + LOG(INFO) << "Driver short name: " + << sandbox.GetCString(ptr_driver_short_name).value().c_str(); + LOG(INFO) << "Driver long name: " + << sandbox.GetCString(ptr_driver_long_name).value().c_str(); + + // Checking that GetGeoTransform is valid. + std::vector adf_geo_transform(6); + sapi::v::Array adf_geo_transform_array(&adf_geo_transform[0], + adf_geo_transform.size()); + + // For this function that returns CPLErr, the error-handling must be done + // analyzing the returning object. + // Same for GDALReturnsIO from below. + CPLErr err; + SAPI_ASSIGN_OR_RETURN(err, api.GDALGetGeoTransform( + &ptr_dataset, adf_geo_transform_array.PtrBoth())); + + // If GDALGetGeoTransform generates an error. + if (err != CE_None) { + return absl::InternalError("GDAL rasterization failed."); + } + + LOG(INFO) << "Origin = (" << adf_geo_transform[0] << ", " + << adf_geo_transform[3] << ")" << std::endl; + LOG(INFO) << "Pixel Size = (" << adf_geo_transform[0] << ", " + << adf_geo_transform[3] << ")" << std::endl; + + std::vector n_blockX_size(1); + std::vector n_blockY_size(1); + + sapi::v::Array nBlockXSizeArray(&n_blockX_size[0], n_blockX_size.size()); + sapi::v::Array nBlockYSizeArray(&n_blockY_size[0], n_blockY_size.size()); + + auto band = api.GDALGetRasterBand(&ptr_dataset, 1); + LOG(INFO) << "Band pointer adress: " << band.value() << std::endl; + if (!band.value()) { + return absl::AbortedError("NULL pointer for Band.\n"); + } + + sapi::v::RemotePtr ptr_band(band.value()); + SAPI_RETURN_IF_ERROR(api.GDALGetBlockSize(&ptr_band, nBlockXSizeArray.PtrBoth(), + nBlockYSizeArray.PtrBoth())); + + LOG(INFO) << "Block = " << n_blockX_size[0] << " x " << n_blockY_size[0] + << std::endl; + + std::vector b_got_min(1); + std::vector b_got_max(1); + + sapi::v::Array b_got_min_array(&b_got_min[0], b_got_min.size()); + sapi::v::Array b_got_max_array(&b_got_max[0], b_got_max.size()); + + auto nX_size = api.GDALGetRasterBandXSize(&ptr_band); + auto nY_size = api.GDALGetRasterBandYSize(&ptr_band); + + std::vector raster_data(nX_size.value() * nY_size.value(), -1); + sapi::v::Array raster_data_array(&raster_data[0], raster_data.size()); + + // We will use CPLErr type of returning value, as before with + // GDALGetGeoTransorm. + SAPI_ASSIGN_OR_RETURN( + err, api.GDALRasterIO(&ptr_band, GF_Read, 0, 0, nX_size.value(), + nY_size.value(), raster_data_array.PtrBoth(), + nX_size.value(), nY_size.value(), GDT_Byte, 0, 0)); + + // If GDALRasterIO generates an error. + if (err != CE_None) { + return absl::InternalError("GDAL rasterization failed."); + } + + std::cout << "Raster data info: " << raster_data_array.ToString() + << std::endl; + + // To print the data content: `std::cout << raster_data_array.GetData() << + // std::endl;` + + return absl::OkStatus(); +} + +int main(int argc, char** argv) { + // The file to be converted should be specified in the first argument while + // running the program. + if (argc < 2) { + std::cout << "You need to provide a file name: ./raster " + "your_tiff_file_absolute_path\n" + "Example: ./raster /usr/home/username/file.tiff" + << std::endl; + return EXIT_FAILURE; + } + + std::ifstream aux_file; + aux_file.open(argv[1]); + if (!aux_file.is_open()) { + std::cout << "Your file name is not valid.\nUnable to open the file." + << std::endl; + return EXIT_FAILURE; + } + std::string filename(argv[1]); + + if (absl::Status status = GdalMain(filename); !status.ok()) { + LOG(ERROR) << "Initialization failed: " << status.ToString(); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} diff --git a/oss-internship-2020/guetzli/BUILD.bazel b/oss-internship-2020/guetzli/BUILD.bazel index b00b12e..f4b4271 100644 --- a/oss-internship-2020/guetzli/BUILD.bazel +++ b/oss-internship-2020/guetzli/BUILD.bazel @@ -24,6 +24,7 @@ cc_library( srcs = ["guetzli_entry_points.cc"], hdrs = ["guetzli_entry_points.h"], deps = [ + "@com_google_absl//absl/status:statusor", "@com_google_sandboxed_api//sandboxed_api:lenval_core", "@com_google_sandboxed_api//sandboxed_api:vars", "@guetzli//:guetzli_lib", @@ -53,9 +54,7 @@ sapi_library( cc_binary( name = "guetzli_sandboxed", srcs = ["guetzli_sandboxed.cc"], - deps = [ - ":guetzli_sapi", - ], + deps = [":guetzli_sapi"], ) cc_test( diff --git a/oss-internship-2020/guetzli/WORKSPACE b/oss-internship-2020/guetzli/WORKSPACE.bazel similarity index 100% rename from oss-internship-2020/guetzli/WORKSPACE rename to oss-internship-2020/guetzli/WORKSPACE.bazel diff --git a/oss-internship-2020/guetzli/guetzli_entry_points.cc b/oss-internship-2020/guetzli/guetzli_entry_points.cc index 8de94b6..98ae384 100644 --- a/oss-internship-2020/guetzli/guetzli_entry_points.cc +++ b/oss-internship-2020/guetzli/guetzli_entry_points.cc @@ -25,8 +25,8 @@ #include "guetzli/jpeg_data_reader.h" #include "guetzli/quality.h" #include "png.h" // NOLINT(build/include) +#include "absl/status/statusor.h" #include "sandboxed_api/sandbox2/util/fileops.h" -#include "sandboxed_api/util/statusor.h" namespace { @@ -51,7 +51,7 @@ sapi::LenValStruct CreateLenValFromData(const void* data, size_t size) { return {size, new_data}; } -sapi::StatusOr ReadFromFd(int fd) { +absl::StatusOr ReadFromFd(int fd) { struct stat file_data; int status = fstat(fd, &file_data); @@ -70,9 +70,9 @@ sapi::StatusOr ReadFromFd(int fd) { return result; } -sapi::StatusOr PrepareDataForProcessing( +absl::StatusOr PrepareDataForProcessing( const ProcessingParams& processing_params) { - sapi::StatusOr input = ReadFromFd(processing_params.remote_fd); + absl::StatusOr input = ReadFromFd(processing_params.remote_fd); if (!input.ok()) { return input.status(); @@ -96,7 +96,7 @@ inline uint8_t BlendOnBlack(const uint8_t val, const uint8_t alpha) { } // Modified version of ReadPNG from original guetzli.cc -sapi::StatusOr ReadPNG(const std::string& data) { +absl::StatusOr ReadPNG(const std::string& data) { std::vector rgb; int xsize, ysize; png_structp png_ptr = diff --git a/oss-internship-2020/guetzli/guetzli_sandboxed.cc b/oss-internship-2020/guetzli/guetzli_sandboxed.cc index 944a572..0e064c7 100644 --- a/oss-internship-2020/guetzli/guetzli_sandboxed.cc +++ b/oss-internship-2020/guetzli/guetzli_sandboxed.cc @@ -17,7 +17,6 @@ #include "guetzli_transaction.h" // NOLINT(build/include) #include "sandboxed_api/sandbox2/util/fileops.h" -#include "sandboxed_api/util/statusor.h" namespace { diff --git a/oss-internship-2020/guetzli/guetzli_sapi_test.cc b/oss-internship-2020/guetzli/guetzli_sapi_test.cc index cd8bc91..b275036 100644 --- a/oss-internship-2020/guetzli/guetzli_sapi_test.cc +++ b/oss-internship-2020/guetzli/guetzli_sapi_test.cc @@ -83,7 +83,7 @@ TEST_F(GuetzliSapiTest, ProcessRGB) { *processing_params.mutable_data() = { in_fd.GetRemoteFd(), 0, kDefaultQualityTarget, kDefaultMemlimitMb}; sapi::v::LenVal output(0); - sapi::StatusOr processing_result = + absl::StatusOr processing_result = api_->ProcessRgb(processing_params.PtrBefore(), output.PtrBoth()); ASSERT_TRUE(processing_result.value_or(false)) << "Error processing rgb data"; std::string reference_data = @@ -108,7 +108,7 @@ TEST_F(GuetzliSapiTest, ProcessJpeg) { *processing_params.mutable_data() = { in_fd.GetRemoteFd(), 0, kDefaultQualityTarget, kDefaultMemlimitMb}; sapi::v::LenVal output(0); - sapi::StatusOr processing_result = + absl::StatusOr processing_result = api_->ProcessJpeg(processing_params.PtrBefore(), output.PtrBoth()); ASSERT_TRUE(processing_result.value_or(false)) << "Error processing jpg data"; std::string reference_data = diff --git a/oss-internship-2020/guetzli/guetzli_transaction.cc b/oss-internship-2020/guetzli/guetzli_transaction.cc index 2bd6e8a..d3bc1bd 100644 --- a/oss-internship-2020/guetzli/guetzli_transaction.cc +++ b/oss-internship-2020/guetzli/guetzli_transaction.cc @@ -99,7 +99,7 @@ absl::Status GuetzliTransaction::LinkOutFile(int out_fd) const { return absl::OkStatus(); } -sapi::StatusOr GuetzliTransaction::GetImageTypeFromFd(int fd) const { +absl::StatusOr GuetzliTransaction::GetImageTypeFromFd(int fd) const { static const unsigned char kPNGMagicBytes[] = { 0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', }; diff --git a/oss-internship-2020/guetzli/guetzli_transaction.h b/oss-internship-2020/guetzli/guetzli_transaction.h index 2996908..2cda32d 100644 --- a/oss-internship-2020/guetzli/guetzli_transaction.h +++ b/oss-internship-2020/guetzli/guetzli_transaction.h @@ -48,7 +48,7 @@ class GuetzliTransaction : public sapi::Transaction { absl::Status Main() final; absl::Status LinkOutFile(int out_fd) const; - sapi::StatusOr GetImageTypeFromFd(int fd) const; + absl::StatusOr GetImageTypeFromFd(int fd) const; const TransactionParams params_; ImageType image_type_ = ImageType::kJpeg; diff --git a/oss-internship-2020/openjpeg/examples/decompress_example.cc b/oss-internship-2020/openjpeg/examples/decompress_example.cc index 4d8630a..a090a22 100644 --- a/oss-internship-2020/openjpeg/examples/decompress_example.cc +++ b/oss-internship-2020/openjpeg/examples/decompress_example.cc @@ -73,12 +73,12 @@ int main(int argc, char* argv[]) { sapi::v::ConstCStr in_file_v(in_file.c_str()); // Initialize library's main data-holders. - sapi::StatusOr stream = + absl::StatusOr stream = api.opj_stream_create_default_file_stream(in_file_v.PtrBefore(), 1); CHECK(stream.ok()) << "Stream initialization failed: " << stream.status(); sapi::v::RemotePtr stream_pointer(stream.value()); - sapi::StatusOr codec = api.opj_create_decompress(OPJ_CODEC_JP2); + absl::StatusOr codec = api.opj_create_decompress(OPJ_CODEC_JP2); CHECK(codec.ok()) << "Codec initialization failed: " << stream.status(); sapi::v::RemotePtr codec_pointer(codec.value()); @@ -86,7 +86,7 @@ int main(int argc, char* argv[]) { status = api.opj_set_default_decoder_parameters(parameters.PtrBoth()); CHECK(status.ok()) << "Parameters initialization failed " << status; - sapi::StatusOr bool_status = + absl::StatusOr bool_status = api.opj_setup_decoder(&codec_pointer, parameters.PtrBefore()); CHECK(bool_status.ok() && bool_status.value()) << "Decoder setup failed"; diff --git a/oss-internship-2020/pffft/master b/oss-internship-2020/pffft/master new file mode 160000 index 0000000..74d7261 --- /dev/null +++ b/oss-internship-2020/pffft/master @@ -0,0 +1 @@ +Subproject commit 74d7261be17cf659d5930d4830609406bd7553e3 diff --git a/sandboxed_api/call.h b/sandboxed_api/call.h index a33306a..06def7e 100644 --- a/sandboxed_api/call.h +++ b/sandboxed_api/call.h @@ -23,8 +23,8 @@ namespace sapi { namespace comms { struct ReallocRequest { - uint64_t old_addr; - uint64_t size; + uintptr_t old_addr; + size_t size; }; // Types of TAGs used with Comms channel. diff --git a/sandboxed_api/client.cc b/sandboxed_api/client.cc index df2b24d..9d490b6 100644 --- a/sandboxed_api/client.cc +++ b/sandboxed_api/client.cc @@ -229,7 +229,7 @@ void HandleCallMsg(const FuncCall& call, FuncRet* ret) { } // Handles requests to allocate memory inside the sandboxee. -void HandleAllocMsg(const uintptr_t size, FuncRet* ret) { +void HandleAllocMsg(const size_t size, FuncRet* ret) { VLOG(1) << "HandleAllocMsg: size=" << size; ret->ret_type = v::Type::kPointer; @@ -237,16 +237,15 @@ void HandleAllocMsg(const uintptr_t size, FuncRet* ret) { // Memory is copied to the pointer using an API that the memory sanitizer // is blind to (process_vm_writev). Initialize the memory here so that // the sandboxed code can still be tested with the memory sanitizer. - ret->int_val = - reinterpret_cast(calloc(1, static_cast(size))); + ret->int_val = reinterpret_cast(calloc(1, size)); #else - ret->int_val = reinterpret_cast(malloc(static_cast(size))); + ret->int_val = reinterpret_cast(malloc(size)); #endif ret->success = true; } // Like HandleAllocMsg(), but handles requests to reallocate memory. -void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) { +void HandleReallocMsg(uintptr_t ptr, size_t size, FuncRet* ret) { VLOG(1) << "HandleReallocMsg(" << absl::StrCat(absl::Hex(ptr)) << ", " << size << ")"; #ifdef MEMORY_SANITIZER @@ -254,8 +253,8 @@ void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) { __sanitizer_get_allocated_size(reinterpret_cast(ptr)); #endif ret->ret_type = v::Type::kPointer; - ret->int_val = reinterpret_cast( - realloc(const_cast(reinterpret_cast(ptr)), size)); + ret->int_val = + reinterpret_cast(realloc(reinterpret_cast(ptr), size)); ret->success = true; #ifdef MEMORY_SANITIZER // Memory is copied to the pointer using an API that the memory sanitizer @@ -273,7 +272,7 @@ void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) { void HandleFreeMsg(uintptr_t ptr, FuncRet* ret) { VLOG(1) << "HandleFreeMsg: free(0x" << absl::StrCat(absl::Hex(ptr)) << ")"; - free(const_cast(reinterpret_cast(ptr))); + free(reinterpret_cast(ptr)); ret->ret_type = v::Type::kVoid; ret->success = true; ret->int_val = 0ULL; @@ -363,14 +362,13 @@ void ServeRequest(sandbox2::Comms* comms) { break; case comms::kMsgAllocate: VLOG(1) << "Client::kMsgAllocate"; - HandleAllocMsg(BytesAs(bytes), &ret); + HandleAllocMsg(BytesAs(bytes), &ret); break; case comms::kMsgReallocate: VLOG(1) << "Client::kMsgReallocate"; { auto req = BytesAs(bytes); - HandleReallocMsg(static_cast(req.old_addr), - static_cast(req.size), &ret); + HandleReallocMsg(req.old_addr, req.size, &ret); } break; case comms::kMsgFree: diff --git a/sandboxed_api/examples/hello_sapi/BUILD.bazel b/sandboxed_api/examples/hello_sapi/BUILD.bazel index d1fd764..edb93a9 100644 --- a/sandboxed_api/examples/hello_sapi/BUILD.bazel +++ b/sandboxed_api/examples/hello_sapi/BUILD.bazel @@ -40,7 +40,6 @@ sapi_library( cc_binary( name = "hello", srcs = ["hello_main.cc"], - includes = ["."], # To find the generated header deps = [":hello_sapi"], ) @@ -50,7 +49,6 @@ cc_binary( cc_binary( name = "hello_transacted", srcs = ["hello_transacted.cc"], - includes = ["."], # To find the generated header deps = [ ":hello_sapi", "@com_google_absl//absl/memory", diff --git a/sandboxed_api/examples/stringop/main_stringop.cc b/sandboxed_api/examples/stringop/main_stringop.cc index 9880699..c7e5b2e 100644 --- a/sandboxed_api/examples/stringop/main_stringop.cc +++ b/sandboxed_api/examples/stringop/main_stringop.cc @@ -134,7 +134,7 @@ TEST(StringopTest, RawStringLength) { ASSERT_THAT(sandbox.Init(), IsOk()); StringopApi api(&sandbox); SAPI_ASSERT_OK_AND_ASSIGN(void* target_mem_ptr, api.get_raw_c_string()); - SAPI_ASSERT_OK_AND_ASSIGN(uint64_t len, + SAPI_ASSERT_OK_AND_ASSIGN(size_t len, sandbox.rpc_channel()->Strlen(target_mem_ptr)); EXPECT_THAT(len, Eq(10)); } @@ -144,7 +144,7 @@ TEST(StringopTest, RawStringReading) { ASSERT_THAT(sandbox.Init(), IsOk()); StringopApi api(&sandbox); SAPI_ASSERT_OK_AND_ASSIGN(void* target_mem_ptr, api.get_raw_c_string()); - SAPI_ASSERT_OK_AND_ASSIGN(uint64_t len, + SAPI_ASSERT_OK_AND_ASSIGN(size_t len, sandbox.rpc_channel()->Strlen(target_mem_ptr)); EXPECT_THAT(len, Eq(10)); diff --git a/sandboxed_api/rpcchannel.cc b/sandboxed_api/rpcchannel.cc index 66e1b2b..72a057e 100644 --- a/sandboxed_api/rpcchannel.cc +++ b/sandboxed_api/rpcchannel.cc @@ -27,8 +27,7 @@ namespace sapi { absl::Status RPCChannel::Call(const FuncCall& call, uint32_t tag, FuncRet* ret, v::Type exp_type) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(tag, sizeof(call), - reinterpret_cast(&call))) { + if (!comms_->SendTLV(tag, sizeof(call), &call)) { return absl::UnavailableError("Sending TLV value failed"); } SAPI_ASSIGN_OR_RETURN(auto fret, Return(exp_type)); @@ -38,7 +37,7 @@ absl::Status RPCChannel::Call(const FuncCall& call, uint32_t tag, FuncRet* ret, absl::StatusOr RPCChannel::Return(v::Type exp_type) { uint32_t tag; - uint64_t len; + size_t len; FuncRet ret; if (!comms_->RecvTLV(&tag, &len, &ret, sizeof(ret))) { return absl::UnavailableError("Receiving TLV value failed"); @@ -67,9 +66,7 @@ absl::StatusOr RPCChannel::Return(v::Type exp_type) { absl::Status RPCChannel::Allocate(size_t size, void** addr) { absl::MutexLock lock(&mutex_); - uint64_t sz = size; - if (!comms_->SendTLV(comms::kMsgAllocate, sizeof(sz), - reinterpret_cast(&sz))) { + if (!comms_->SendTLV(comms::kMsgAllocate, sizeof(size), &size)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -81,12 +78,13 @@ absl::Status RPCChannel::Allocate(size_t size, void** addr) { absl::Status RPCChannel::Reallocate(void* old_addr, size_t size, void** new_addr) { absl::MutexLock lock(&mutex_); - comms::ReallocRequest req; - req.old_addr = reinterpret_cast(old_addr); - req.size = size; + comms::ReallocRequest req = { + .old_addr = reinterpret_cast(old_addr), + .size = size, + }; if (!comms_->SendTLV(comms::kMsgReallocate, sizeof(comms::ReallocRequest), - reinterpret_cast(&req))) { + &req)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -103,9 +101,8 @@ absl::Status RPCChannel::Reallocate(void* old_addr, size_t size, absl::Status RPCChannel::Free(void* addr) { absl::MutexLock lock(&mutex_); - uint64_t remote = reinterpret_cast(addr); - if (!comms_->SendTLV(comms::kMsgFree, sizeof(remote), - reinterpret_cast(&remote))) { + uintptr_t remote = reinterpret_cast(addr); + if (!comms_->SendTLV(comms::kMsgFree, sizeof(remote), &remote)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -118,8 +115,7 @@ absl::Status RPCChannel::Free(void* addr) { absl::Status RPCChannel::Symbol(const char* symname, void** addr) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(comms::kMsgSymbol, strlen(symname) + 1, - reinterpret_cast(symname))) { + if (!comms_->SendTLV(comms::kMsgSymbol, strlen(symname) + 1, symname)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -137,9 +133,8 @@ absl::Status RPCChannel::Exit() { // Try the RPC exit sequence. But, the only thing that matters as a success // indicator is whether the Comms channel had been closed - bool unused = true; - comms_->SendTLV(comms::kMsgExit, sizeof(unused), - reinterpret_cast(&unused)); + comms_->SendTLV(comms::kMsgExit, 0, nullptr); + bool unused; comms_->RecvBool(&unused); if (!comms_->IsTerminated()) { @@ -154,9 +149,7 @@ absl::Status RPCChannel::Exit() { absl::Status RPCChannel::SendFD(int local_fd, int* remote_fd) { absl::MutexLock lock(&mutex_); - bool unused = true; - if (!comms_->SendTLV(comms::kMsgSendFd, sizeof(unused), - reinterpret_cast(&unused))) { + if (!comms_->SendTLV(comms::kMsgSendFd, 0, nullptr)) { return absl::UnavailableError("Sending TLV value failed"); } if (!comms_->SendFD(local_fd)) { @@ -173,8 +166,7 @@ absl::Status RPCChannel::SendFD(int local_fd, int* remote_fd) { absl::Status RPCChannel::RecvFD(int remote_fd, int* local_fd) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(comms::kMsgRecvFd, sizeof(remote_fd), - reinterpret_cast(&remote_fd))) { + if (!comms_->SendTLV(comms::kMsgRecvFd, sizeof(remote_fd), &remote_fd)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -191,8 +183,7 @@ absl::Status RPCChannel::RecvFD(int remote_fd, int* local_fd) { absl::Status RPCChannel::Close(int remote_fd) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(comms::kMsgClose, sizeof(remote_fd), - reinterpret_cast(&remote_fd))) { + if (!comms_->SendTLV(comms::kMsgClose, sizeof(remote_fd), &remote_fd)) { return absl::UnavailableError("Sending TLV value failed"); } @@ -203,10 +194,9 @@ absl::Status RPCChannel::Close(int remote_fd) { return absl::OkStatus(); } -absl::StatusOr RPCChannel::Strlen(void* str) { +absl::StatusOr RPCChannel::Strlen(void* str) { absl::MutexLock lock(&mutex_); - if (!comms_->SendTLV(comms::kMsgStrlen, sizeof(str), - reinterpret_cast(&str))) { + if (!comms_->SendTLV(comms::kMsgStrlen, sizeof(str), &str)) { return absl::UnavailableError("Sending TLV value failed"); } diff --git a/sandboxed_api/rpcchannel.h b/sandboxed_api/rpcchannel.h index 4fec23b..56d7ed8 100644 --- a/sandboxed_api/rpcchannel.h +++ b/sandboxed_api/rpcchannel.h @@ -61,7 +61,7 @@ class RPCChannel { absl::Status Close(int remote_fd); // Returns length of a null-terminated c-style string (invokes strlen). - absl::StatusOr Strlen(void* str); + absl::StatusOr Strlen(void* str); sandbox2::Comms* comms() const { return comms_; } diff --git a/sandboxed_api/sandbox.cc b/sandboxed_api/sandbox.cc index 8c9da3d..d6b1bf2 100644 --- a/sandboxed_api/sandbox.cc +++ b/sandboxed_api/sandbox.cc @@ -394,7 +394,7 @@ absl::Status Sandbox::TransferFromSandboxee(v::Var* var) { } absl::StatusOr Sandbox::GetCString(const v::RemotePtr& str, - uint64_t max_length) { + size_t max_length) { if (!is_active()) { return absl::UnavailableError("Sandbox not active"); } diff --git a/sandboxed_api/sandbox.h b/sandboxed_api/sandbox.h index 5d69eb0..576a144 100644 --- a/sandboxed_api/sandbox.h +++ b/sandboxed_api/sandbox.h @@ -103,8 +103,8 @@ class Sandbox { absl::Status TransferFromSandboxee(v::Var* var); absl::StatusOr GetCString(const v::RemotePtr& str, - uint64_t max_length = 10ULL - << 20 /* 10 MiB*/ + size_t max_length = 10ULL + << 20 /* 10 MiB*/ ); // Waits until the sandbox terminated and returns the result. diff --git a/sandboxed_api/sandbox2/comms.cc b/sandboxed_api/sandbox2/comms.cc index 5195a9d..449db8f 100644 --- a/sandboxed_api/sandbox2/comms.cc +++ b/sandboxed_api/sandbox2/comms.cc @@ -228,7 +228,7 @@ void Comms::Terminate() { } } -bool Comms::SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes) { +bool Comms::SendTLV(uint32_t tag, size_t length, const void* value) { if (length > GetMaxMsgSize()) { SAPI_RAW_LOG(ERROR, "Maximum TLV message size exceeded: (%u > %u)", length, GetMaxMsgSize()); @@ -249,14 +249,14 @@ bool Comms::SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes) { length); { absl::MutexLock lock(&tlv_send_transmission_mutex_); - if (!Send(reinterpret_cast(&tag), sizeof(tag))) { + if (!Send(&tag, sizeof(tag))) { return false; } - if (!Send(reinterpret_cast(&length), sizeof(length))) { + if (!Send(&length, sizeof(length))) { return false; } if (length > 0) { - if (!Send(bytes, length)) { + if (!Send(value, length)) { return false; } } @@ -279,8 +279,7 @@ bool Comms::RecvString(std::string* v) { } bool Comms::SendString(const std::string& v) { - return SendTLV(kTagString, v.length(), - reinterpret_cast(v.c_str())); + return SendTLV(kTagString, v.length(), v.c_str()); } bool Comms::RecvBytes(std::vector* buffer) { @@ -297,7 +296,7 @@ bool Comms::RecvBytes(std::vector* buffer) { return true; } -bool Comms::SendBytes(const uint8_t* v, uint64_t len) { +bool Comms::SendBytes(const uint8_t* v, size_t len) { return SendTLV(kTagBytes, len, v); } @@ -418,7 +417,7 @@ bool Comms::SendFD(int fd) { int* fds = reinterpret_cast(CMSG_DATA(cmsg)); fds[0] = fd; - InternalTLV tlv = {kTagFd, sizeof(tlv.val), 0}; + InternalTLV tlv = {kTagFd, 0}; iovec iov; iov.iov_base = &tlv; @@ -513,8 +512,9 @@ socklen_t Comms::CreateSockaddrUn(sockaddr_un* sun) { return slen; } -bool Comms::Send(const uint8_t* bytes, uint64_t len) { - uint64_t total_sent = 0; +bool Comms::Send(const void* data, size_t len) { + size_t total_sent = 0; + const char* bytes = reinterpret_cast(data); const auto op = [bytes, len, &total_sent](int fd) -> ssize_t { PotentiallyBlockingRegion region; return TEMP_FAILURE_RETRY(write(fd, &bytes[total_sent], len - total_sent)); @@ -545,8 +545,9 @@ bool Comms::Send(const uint8_t* bytes, uint64_t len) { return true; } -bool Comms::Recv(uint8_t* bytes, uint64_t len) { - uint64_t total_recv = 0; +bool Comms::Recv(void* data, size_t len) { + size_t total_recv = 0; + char* bytes = reinterpret_cast(data); const auto op = [bytes, len, &total_recv](int fd) -> ssize_t { PotentiallyBlockingRegion region; return TEMP_FAILURE_RETRY(read(fd, &bytes[total_recv], len - total_recv)); @@ -573,7 +574,7 @@ bool Comms::Recv(uint8_t* bytes, uint64_t len) { } // Internal helper method (low level). -bool Comms::RecvTL(uint32_t* tag, uint64_t* length) { +bool Comms::RecvTL(uint32_t* tag, size_t* length) { if (!Recv(reinterpret_cast(tag), sizeof(*tag))) { return false; } @@ -610,7 +611,7 @@ bool Comms::RecvTLV(uint32_t* tag, std::string* value) { template bool Comms::RecvTLVGeneric(uint32_t* tag, T* value) { absl::MutexLock lock(&tlv_recv_transmission_mutex_); - uint64_t length; + size_t length; if (!RecvTL(tag, &length)) { return false; } @@ -619,28 +620,29 @@ bool Comms::RecvTLVGeneric(uint32_t* tag, T* value) { return length == 0 || Recv(reinterpret_cast(value->data()), length); } -bool Comms::RecvTLV(uint32_t* tag, uint64_t* length, void* buffer, - uint64_t buffer_size) { +bool Comms::RecvTLV(uint32_t* tag, size_t* length, void* buffer, + size_t buffer_size) { absl::MutexLock lock(&tlv_recv_transmission_mutex_); if (!RecvTL(tag, length)) { return false; } + if (*length == 0) { + return true; + } + if (*length > buffer_size) { SAPI_RAW_LOG(ERROR, "Buffer size too small (0x%x > 0x%x)", *length, buffer_size); return false; - } else if (*length > 0) { - if (!Recv(reinterpret_cast(buffer), *length)) { - return false; - } } - return true; + + return Recv(reinterpret_cast(buffer), *length); } -bool Comms::RecvInt(void* buffer, uint64_t len, uint32_t tag) { +bool Comms::RecvInt(void* buffer, size_t len, uint32_t tag) { uint32_t received_tag; - uint64_t received_length; + size_t received_length; if (!RecvTLV(&received_tag, &received_length, buffer, len)) { return false; } diff --git a/sandboxed_api/sandbox2/comms.h b/sandboxed_api/sandbox2/comms.h index 7863288..fca8c53 100644 --- a/sandboxed_api/sandbox2/comms.h +++ b/sandboxed_api/sandbox2/comms.h @@ -60,7 +60,7 @@ class Comms { static constexpr uint32_t kTagFd = 0X80000201; // Any payload size above this limit will LOG(WARNING). - static constexpr uint64_t kWarnMsgSize = (256ULL << 20); + static constexpr size_t kWarnMsgSize = (256ULL << 20); // Sandbox2-specific convention where FD=1023 is always passed to the // sandboxed process as a communication channel (encapsulated in the @@ -103,9 +103,9 @@ class Comms { // Note: The actual size is "unlimited", although the Buffer API is more // efficient for large transfers. There is an arbitrary limit to ~2GiB to // avoid protobuf serialization issues. - uint64_t GetMaxMsgSize() const { return std::numeric_limits::max(); } + size_t GetMaxMsgSize() const { return std::numeric_limits::max(); } - bool SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes); + bool SendTLV(uint32_t tag, size_t length, const void* value); // Receive a TLV structure, the memory for the value will be allocated // by std::vector. bool RecvTLV(uint32_t* tag, std::vector* value); @@ -113,7 +113,7 @@ class Comms { // by std::string. bool RecvTLV(uint32_t* tag, std::string* value); // Receives a TLV value into a specified buffer without allocating memory. - bool RecvTLV(uint32_t* tag, uint64_t* length, void* buffer, uint64_t buffer_size); + bool RecvTLV(uint32_t* tag, size_t* length, void* buffer, size_t buffer_size); // Sends/receives various types of data. bool RecvUint8(uint8_t* v) { return RecvIntGeneric(v, kTagUint8); } @@ -138,7 +138,7 @@ class Comms { bool SendString(const std::string& v); bool RecvBytes(std::vector* buffer); - bool SendBytes(const uint8_t* v, uint64_t len); + bool SendBytes(const uint8_t* v, size_t len); bool SendBytes(const std::vector& buffer); // Receives remote process credentials. @@ -183,19 +183,18 @@ class Comms { struct ABSL_ATTRIBUTE_PACKED InternalTLV { uint32_t tag; uint32_t len; - uint64_t val; }; // Fills sockaddr_un struct with proper values. socklen_t CreateSockaddrUn(sockaddr_un* sun); // Support for EINTR and size completion. - bool Send(const uint8_t* bytes, uint64_t len); - bool Recv(uint8_t* bytes, uint64_t len); + bool Send(const void* data, size_t len); + bool Recv(void* data, size_t len); // Receives tag and length. Assumes that the `tlv_transmission_mutex_` mutex // is locked. - bool RecvTL(uint32_t* tag, uint64_t* length) + bool RecvTL(uint32_t* tag, size_t* length) ABSL_EXCLUSIVE_LOCKS_REQUIRED(tlv_recv_transmission_mutex_); // T has to be a ContiguousContainer @@ -203,7 +202,7 @@ class Comms { bool RecvTLVGeneric(uint32_t* tag, T* value); // Receives arbitrary integers. - bool RecvInt(void* buffer, uint64_t len, uint32_t tag); + bool RecvInt(void* buffer, size_t len, uint32_t tag); template bool RecvIntGeneric(T* output, uint32_t tag) { @@ -212,7 +211,7 @@ class Comms { template bool SendGeneric(T value, uint32_t tag) { - return SendTLV(tag, sizeof(T), reinterpret_cast(&value)); + return SendTLV(tag, sizeof(T), &value); } }; diff --git a/sandboxed_api/tools/clang_generator/CMakeLists.txt b/sandboxed_api/tools/clang_generator/CMakeLists.txt index c134e76..e3fd9c2 100644 --- a/sandboxed_api/tools/clang_generator/CMakeLists.txt +++ b/sandboxed_api/tools/clang_generator/CMakeLists.txt @@ -45,6 +45,7 @@ target_link_libraries(sapi_generator PUBLIC absl::memory absl::random_random absl::status + absl::statusor absl::strings clangFormat clangFrontendTool @@ -59,12 +60,15 @@ add_executable(sapi_generator_tool ) target_link_libraries(sapi_generator_tool PRIVATE sapi::base + sandbox2::file_helpers sandbox2::fileops sapi::generator ) if(SAPI_ENABLE_TESTS) add_executable(sapi_generator_test + frontend_action_test_util.cc + frontend_action_test_util.h emitter_test.cc ) target_link_libraries(sapi_generator_test PRIVATE diff --git a/sandboxed_api/tools/clang_generator/emitter.cc b/sandboxed_api/tools/clang_generator/emitter.cc index 03bb5e3..a8c1420 100644 --- a/sandboxed_api/tools/clang_generator/emitter.cc +++ b/sandboxed_api/tools/clang_generator/emitter.cc @@ -23,9 +23,13 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" #include "absl/strings/strip.h" #include "clang/AST/DeclCXX.h" +#include "clang/AST/Type.h" +#include "clang/Format/Format.h" #include "sandboxed_api/tools/clang_generator/diagnostics.h" +#include "sandboxed_api/tools/clang_generator/generator.h" #include "sandboxed_api/util/status_macros.h" namespace sapi { @@ -106,6 +110,30 @@ constexpr absl::string_view kClassFooterTemplate = R"( }; )"; +namespace internal { + +absl::StatusOr ReformatGoogleStyle(const std::string& filename, + const std::string& code) { + // Configure code style based on Google style, but enforce pointer alignment + clang::format::FormatStyle style = + clang::format::getGoogleStyle(clang::format::FormatStyle::LK_Cpp); + style.DerivePointerAlignment = false; + style.PointerAlignment = clang::format::FormatStyle::PAS_Left; + + clang::tooling::Replacements replacements = clang::format::reformat( + style, code, llvm::makeArrayRef(clang::tooling::Range(0, code.size())), + filename); + + llvm::Expected formatted_header = + clang::tooling::applyAllReplacements(code, replacements); + if (!formatted_header) { + return absl::InternalError(llvm::toString(formatted_header.takeError())); + } + return *formatted_header; +} + +} // namespace internal + std::string GetIncludeGuard(absl::string_view filename) { if (filename.empty()) { static auto* bit_gen = new absl::BitGen(); @@ -250,7 +278,8 @@ absl::StatusOr EmitFunction(const clang::FunctionDecl* decl) { } absl::StatusOr EmitHeader( - std::vector functions, const QualTypeSet& types, + const std::vector& functions, + const Emitter::RenderedTypesMap& rendered_types, const GeneratorOptions& options) { std::string out; const std::string include_guard = GetIncludeGuard(options.out_file); @@ -276,45 +305,19 @@ absl::StatusOr EmitHeader( } // Emit type dependencies - // TODO(cblichmann): Coalesce namespaces - std::string out_types = "// Types this API depends on\n"; - bool added_types = false; - for (const clang::QualType& qual : types) { - clang::TypeDecl* decl = nullptr; - if (const auto* typedef_type = qual->getAs()) { - decl = typedef_type->getDecl(); - } else if (const auto* enum_type = qual->getAs()) { - decl = enum_type->getDecl(); - } else { - decl = qual->getAsRecordDecl(); - } - if (!decl) { - continue; - } - - const std::vector ns_path = GetNamespacePath(decl); - std::string nested_ns_name; - if (!ns_path.empty()) { - if (const auto& ns_root = ns_path.front(); - ns_root == "std" || ns_root == "sapi" || ns_root == "__gnu_cxx") { - // Filter out any and all declarations from the C++ standard library, - // from SAPI itself and from other well-known namespaces. This avoids - // re-declaring things like standard integer types, for example. - continue; + if (!rendered_types.empty()) { + absl::StrAppend(&out, "// Types this API depends on\n"); + for (const auto& [ns_name, types] : rendered_types) { + if (!ns_name.empty()) { + absl::StrAppend(&out, "namespace ", ns_name, " {\n"); + } + for (const auto& type : types) { + absl::StrAppend(&out, type, ";\n"); + } + if (!ns_name.empty()) { + absl::StrAppend(&out, "} // namespace ", ns_name, "\n\n"); } - nested_ns_name = absl::StrCat(ns_path[0].empty() ? "" : " ", - absl::StrJoin(ns_path, "::")); - absl::StrAppend(&out_types, "namespace", nested_ns_name, " {\n"); } - absl::StrAppend(&out_types, PrintAstDecl(decl), ";"); - if (!ns_path.empty()) { - absl::StrAppend(&out_types, "\n} // namespace", nested_ns_name); - } - absl::StrAppend(&out_types, "\n"); - added_types = true; - } - if (added_types) { - absl::StrAppend(&out, out_types); } // Optionally emit a default sandbox that instantiates an embedded sandboxee @@ -329,11 +332,7 @@ absl::StatusOr EmitHeader( // TODO(cblichmann): Make the "Api" suffix configurable or at least optional. absl::StrAppendFormat(&out, kClassHeaderTemplate, absl::StrCat(options.name, "Api")); - std::string out_func; - for (const clang::FunctionDecl* decl : functions) { - SAPI_ASSIGN_OR_RETURN(out_func, EmitFunction(decl)); - absl::StrAppend(&out, out_func); - } + absl::StrAppend(&out, absl::StrJoin(functions, "\n")); absl::StrAppend(&out, kClassFooterTemplate); // Close out the header: close namespace (if needed) and end include guard @@ -344,4 +343,45 @@ absl::StatusOr EmitHeader( return out; } +void Emitter::CollectType(clang::QualType qual) { + clang::TypeDecl* decl = nullptr; + if (const auto* typedef_type = qual->getAs()) { + decl = typedef_type->getDecl(); + } else if (const auto* enum_type = qual->getAs()) { + decl = enum_type->getDecl(); + } else { + decl = qual->getAsRecordDecl(); + } + if (!decl) { + return; + } + + const std::vector ns_path = GetNamespacePath(decl); + std::string ns_name; + if (!ns_path.empty()) { + if (const auto& ns_root = ns_path.front(); + ns_root == "std" || ns_root == "sapi" || ns_root == "__gnu_cxx") { + // Filter out any and all declarations from the C++ standard library, + // from SAPI itself and from other well-known namespaces. This avoids + // re-declaring things like standard integer types, for example. + return; + } + ns_name = absl::StrCat(ns_path[0].empty() ? "" : " ", + absl::StrJoin(ns_path, "::")); + } + + rendered_types_[ns_name].push_back(PrintAstDecl(decl)); +} + +void Emitter::CollectFunction(clang::FunctionDecl* decl) { + functions_.push_back(*EmitFunction(decl)); // Cannot currently fail +} + +absl::StatusOr Emitter::EmitHeader( + const GeneratorOptions& options) { + SAPI_ASSIGN_OR_RETURN(const std::string header, + ::sapi::EmitHeader(functions_, rendered_types_, options)); + return internal::ReformatGoogleStyle(options.out_file, header); +} + } // namespace sapi diff --git a/sandboxed_api/tools/clang_generator/emitter.h b/sandboxed_api/tools/clang_generator/emitter.h index 78e1f12..6932698 100644 --- a/sandboxed_api/tools/clang_generator/emitter.h +++ b/sandboxed_api/tools/clang_generator/emitter.h @@ -16,16 +16,44 @@ #define SANDBOXED_API_TOOLS_CLANG_GENERATOR_EMITTER_H_ #include +#include +#include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "clang/AST/Decl.h" #include "clang/AST/Type.h" -#include "sandboxed_api/tools/clang_generator/generator.h" #include "sandboxed_api/tools/clang_generator/types.h" namespace sapi { +namespace internal { + +absl::StatusOr ReformatGoogleStyle(const std::string& filename, + const std::string& code); + +} // namespace internal + +class GeneratorOptions; + +class Emitter { + public: + using RenderedTypesMap = + absl::flat_hash_map>; + + void CollectType(clang::QualType qual); + void CollectFunction(clang::FunctionDecl* decl); + + // Outputs a formatted header for a list of functions and their related types. + absl::StatusOr EmitHeader(const GeneratorOptions& options); + + protected: + // Maps namespace to a list of spellings for types + RenderedTypesMap rendered_types_; + + // Functions for sandboxed API, including their bodies + std::vector functions_; +}; // Constructs an include guard name for the given filename. The name is of the // same form as the include guards in this project. @@ -35,11 +63,6 @@ namespace sapi { // SANDBOXED_API_EXAMPLES_ZLIB_ZLIB_SAPI_SAPI_H_ std::string GetIncludeGuard(absl::string_view filename); -// Outputs a formatted header for a list of functions and their related types. -absl::StatusOr EmitHeader( - std::vector functions, const QualTypeSet& types, - const GeneratorOptions& options); - } // namespace sapi #endif // SANDBOXED_API_TOOLS_CLANG_GENERATOR_EMITTER_H_ diff --git a/sandboxed_api/tools/clang_generator/emitter_test.cc b/sandboxed_api/tools/clang_generator/emitter_test.cc index 7956df1..e6e82e9 100644 --- a/sandboxed_api/tools/clang_generator/emitter_test.cc +++ b/sandboxed_api/tools/clang_generator/emitter_test.cc @@ -14,18 +14,48 @@ #include "sandboxed_api/tools/clang_generator/emitter.h" +#include + #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/memory/memory.h" #include "absl/strings/string_view.h" +#include "sandboxed_api/sandbox2/testing.h" +#include "sandboxed_api/tools/clang_generator/frontend_action_test_util.h" +#include "sandboxed_api/tools/clang_generator/generator.h" #include "sandboxed_api/util/status_matchers.h" namespace sapi { namespace { using ::testing::MatchesRegex; +using ::testing::SizeIs; using ::testing::StrEq; using ::testing::StrNe; +class EmitterForTesting : public Emitter { + public: + using Emitter::functions_; +}; + +class EmitterTest : public FrontendActionTest {}; + +TEST_F(EmitterTest, BasicFunctionality) { + GeneratorOptions options; + options.out_file = "input.h"; + options.set_function_names>( + {"ExposedFunction"}); + + EmitterForTesting emitter; + RunFrontendAction(R"(extern "C" void ExposedFunction() {})", + absl::make_unique(emitter, options)); + + EXPECT_THAT(emitter.functions_, SizeIs(1)); + + absl::StatusOr header = emitter.EmitHeader(options); + EXPECT_THAT(header, IsOk()); +} + TEST(IncludeGuard, CreatesRandomizedGuardForEmptyFilename) { // Copybara will transform the string. This is intentional. constexpr absl::string_view kGeneratedHeaderPrefix = diff --git a/sandboxed_api/tools/clang_generator/frontend_action_test_util.cc b/sandboxed_api/tools/clang_generator/frontend_action_test_util.cc new file mode 100644 index 0000000..490ed65 --- /dev/null +++ b/sandboxed_api/tools/clang_generator/frontend_action_test_util.cc @@ -0,0 +1,70 @@ +// Copyright 2020 google llc +// +// licensed under the apache license, version 2.0 (the "license"); +// you may not use this file except in compliance with the license. +// you may obtain a copy of the license at +// +// http://www.apache.org/licenses/license-2.0 +// +// unless required by applicable law or agreed to in writing, software +// distributed under the license is distributed on an "as is" basis, +// without warranties or conditions of any kind, either express or implied. +// see the license for the specific language governing permissions and +// limitations under the license. + +#include "sandboxed_api/tools/clang_generator/frontend_action_test_util.h" + +#include + +#include "absl/status/status.h" +#include "clang/Basic/FileManager.h" +#include "clang/Basic/FileSystemOptions.h" +#include "clang/Frontend/FrontendAction.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/VirtualFileSystem.h" + +namespace sapi { +namespace internal { + +absl::Status RunClangTool( + const std::vector command_line, + const absl::flat_hash_map file_contents, + std::unique_ptr action) { + // Setup an in-memory virtual filesystem + llvm::IntrusiveRefCntPtr fs( + new llvm::vfs::InMemoryFileSystem()); + llvm::IntrusiveRefCntPtr files = + new clang::FileManager(clang::FileSystemOptions(), fs); + + for (const auto& [filename, content] : file_contents) { + if (!fs->addFile(filename, /*ModificationTime=*/0, + llvm::MemoryBuffer::getMemBuffer(content))) { + return absl::UnknownError( + absl::StrCat("Couldn't add file to in-memory VFS: ", filename)); + } + } + +#if LLVM_VERSION_MAJOR >= 10 + clang::tooling::ToolInvocation invocation(command_line, std::move(action), + files.get()); +#else + clang::tooling::ToolInvocation invocation(command_line, action.get(), + files.get()); +#endif + if (!invocation.run()) { + return absl::UnknownError("Tool invocation failed"); + } + return absl::OkStatus(); +} + +} // namespace internal + +std::vector FrontendActionTest::GetCommandLineFlagsForTesting( + absl::string_view input_file) { + return {"tool", "-fsyntax-only", "--std=c++17", + "-I.", "-Wno-error", std::string(input_file)}; +} + +} // namespace sapi diff --git a/sandboxed_api/tools/clang_generator/frontend_action_test_util.h b/sandboxed_api/tools/clang_generator/frontend_action_test_util.h new file mode 100644 index 0000000..b0730ef --- /dev/null +++ b/sandboxed_api/tools/clang_generator/frontend_action_test_util.h @@ -0,0 +1,83 @@ +// Copyright 2020 google llc +// +// licensed under the apache license, version 2.0 (the "license"); +// you may not use this file except in compliance with the license. +// you may obtain a copy of the license at +// +// http://www.apache.org/licenses/license-2.0 +// +// unless required by applicable law or agreed to in writing, software +// distributed under the license is distributed on an "as is" basis, +// without warranties or conditions of any kind, either express or implied. +// see the license for the specific language governing permissions and +// limitations under the license. + +#ifndef SANDBOXED_API_TOOLS_CLANG_GENERATOR_FRONTEND_ACTION_TEST_UTIL_H_ +#define SANDBOXED_API_TOOLS_CLANG_GENERATOR_FRONTEND_ACTION_TEST_UTIL_H_ + +#include +#include +#include + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/flat_hash_map.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "clang/Frontend/FrontendAction.h" +#include "sandboxed_api/util/status_matchers.h" + +namespace sapi { +namespace internal { + +absl::Status RunClangTool( + const std::vector command_line, + const absl::flat_hash_map file_contents, + std::unique_ptr action); + +} // namespace internal + +class FrontendActionTest : public ::testing::Test { + protected: + // Adds code to a virtual filesystem with the given filename. + void AddCode(const std::string& filename, absl::string_view code) { + absl::StrAppend(&file_contents_[filename], code); + } + + // Changes the name of the virtual input file. Useful for special cases where + // the filenames of compiled sources matter. + void set_input_file(absl::string_view value) { + input_file_ = std::string(value); + } + + virtual std::vector GetCommandLineFlagsForTesting( + absl::string_view input_file); + + // Runs the specified frontend action on in-memory source code. + void RunFrontendAction(absl::string_view code, + std::unique_ptr action) { + std::vector command_line = + GetCommandLineFlagsForTesting(input_file_); + AddCode(input_file_, code); + ASSERT_THAT( + internal::RunClangTool(command_line, file_contents_, std::move(action)), + IsOk()); + } + + // Runs the specified frontend action. Provided for compatibility with LLVM < + // 10. Takes ownership. + void RunFrontendAction(absl::string_view code, + clang::FrontendAction* action) { + RunFrontendAction(code, absl::WrapUnique(action)); + } + + private: + std::string input_file_ = "input.cc"; + absl::flat_hash_map file_contents_; +}; + +} // namespace sapi + +#endif // SANDBOXED_API_TOOLS_CLANG_GENERATOR_FRONTEND_ACTION_TEST_UTIL_H_ diff --git a/sandboxed_api/tools/clang_generator/generator.cc b/sandboxed_api/tools/clang_generator/generator.cc index 90504eb..091b234 100644 --- a/sandboxed_api/tools/clang_generator/generator.cc +++ b/sandboxed_api/tools/clang_generator/generator.cc @@ -19,6 +19,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "clang/AST/Type.h" #include "clang/Format/Format.h" #include "sandboxed_api/sandbox2/util/fileops.h" #include "sandboxed_api/tools/clang_generator/diagnostics.h" @@ -39,88 +40,46 @@ std::string ReplaceFileExtension(absl::string_view path, return absl::StrCat(path.substr(0, pos), new_extension); } -std::string GetOutputFilename(absl::string_view source_file) { - return ReplaceFileExtension(source_file, ".sapi.h"); -} - inline absl::string_view ToStringView(llvm::StringRef ref) { return absl::string_view(ref.data(), ref.size()); } } // namespace +std::string GetOutputFilename(absl::string_view source_file) { + return ReplaceFileExtension(source_file, ".sapi.h"); +} + bool GeneratorASTVisitor::VisitFunctionDecl(clang::FunctionDecl* decl) { if (!decl->isCXXClassMember() && // Skip classes decl->isExternC() && // Skip non external functions !decl->isTemplated() && // Skip function templates // Process either all function or just the requested ones - (options_->function_names.empty() || - options_->function_names.count(ToStringView(decl->getName())) > 0)) { + (options_.function_names.empty() || + options_.function_names.count(ToStringView(decl->getName())) > 0)) { functions_.push_back(decl); - GatherRelatedTypes(decl->getDeclaredReturnType(), &types_); + + CollectRelatedTypes(decl->getDeclaredReturnType(), &types_); for (const clang::ParmVarDecl* param : decl->parameters()) { - GatherRelatedTypes(param->getType(), &types_); + CollectRelatedTypes(param->getType(), &types_); } } return true; } -namespace internal { - -absl::StatusOr ReformatGoogleStyle(const std::string& filename, - const std::string& code) { - // Configure code style based on Google style, but enforce pointer alignment - clang::format::FormatStyle style = - clang::format::getGoogleStyle(clang::format::FormatStyle::LK_Cpp); - style.DerivePointerAlignment = false; - style.PointerAlignment = clang::format::FormatStyle::PAS_Left; - - clang::tooling::Replacements replacements = clang::format::reformat( - style, code, llvm::makeArrayRef(clang::tooling::Range(0, code.size())), - filename); - - llvm::Expected formatted_header = - clang::tooling::applyAllReplacements(code, replacements); - if (!formatted_header) { - return absl::InternalError(llvm::toString(formatted_header.takeError())); - } - return *formatted_header; -} - -} // namespace internal - -absl::Status GeneratorASTConsumer::GenerateAndSaveHeader() { - const std::string out_file = - options_->out_file.empty() ? GetOutputFilename(in_file_) - : sandbox2::file_util::fileops::MakeAbsolute( - options_->out_file, options_->work_dir); - - SAPI_ASSIGN_OR_RETURN(const std::string header, - EmitHeader(visitor_.functions_, visitor_.types_, *options_)); - SAPI_ASSIGN_OR_RETURN(const std::string formatted_header, - internal::ReformatGoogleStyle(in_file_, header)); - - std::ofstream os(out_file, std::ios::out | std::ios::trunc); - os << formatted_header; - if (!os) { - return absl::UnknownError("I/O error"); - } - return absl::OkStatus(); -} - void GeneratorASTConsumer::HandleTranslationUnit(clang::ASTContext& context) { - absl::Status status; + std::cout << "Processing " << in_file_ << "\n"; if (!visitor_.TraverseDecl(context.getTranslationUnitDecl())) { - status = absl::InternalError("AST traversal exited early"); - } else { - status = GenerateAndSaveHeader(); + ReportFatalError(context.getDiagnostics(), + context.getTranslationUnitDecl()->getBeginLoc(), + "AST traversal exited early"); } - if (!status.ok()) { - ReportFatalError(context.getDiagnostics(), - GetDiagnosticLocationFromStatus(status).value_or( - context.getTranslationUnitDecl()->getBeginLoc()), - status.message()); + for (clang::QualType qual : visitor_.types_) { + emitter_.CollectType(qual); + } + for (clang::FunctionDecl* func : visitor_.functions_) { + emitter_.CollectFunction(func); } } diff --git a/sandboxed_api/tools/clang_generator/generator.h b/sandboxed_api/tools/clang_generator/generator.h index 8732a26..bc59860 100644 --- a/sandboxed_api/tools/clang_generator/generator.h +++ b/sandboxed_api/tools/clang_generator/generator.h @@ -26,6 +26,7 @@ #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendAction.h" #include "clang/Tooling/Tooling.h" +#include "sandboxed_api/tools/clang_generator/emitter.h" #include "sandboxed_api/tools/clang_generator/types.h" namespace sapi { @@ -46,85 +47,86 @@ struct GeneratorOptions { std::string work_dir; std::string name; // Name of the Sandboxed API std::string namespace_name; // Namespace to wrap the SAPI in - std::string out_file; // Output path of the generated header - std::string embed_dir; // Directory with embedded includes - std::string embed_name; // Identifier of the embed object + // Output path of the generated header. Used to build the header include + // guard. + std::string out_file; + std::string embed_dir; // Directory with embedded includes + std::string embed_name; // Identifier of the embed object }; class GeneratorASTVisitor : public clang::RecursiveASTVisitor { public: + explicit GeneratorASTVisitor(const GeneratorOptions& options) + : options_(options) {} + bool VisitFunctionDecl(clang::FunctionDecl* decl); private: friend class GeneratorASTConsumer; - const GeneratorOptions* options_ = nullptr; std::vector functions_; QualTypeSet types_; + + const GeneratorOptions& options_; }; -namespace internal { - -absl::StatusOr ReformatGoogleStyle(const std::string& filename, - const std::string& code); - -} // namespace internal - class GeneratorASTConsumer : public clang::ASTConsumer { public: - GeneratorASTConsumer(std::string in_file, const GeneratorOptions* options) - : in_file_(std::move(in_file)), options_(options) { - visitor_.options_ = options_; - } + GeneratorASTConsumer(std::string in_file, Emitter& emitter, + const GeneratorOptions& options) + : in_file_(std::move(in_file)), visitor_(options), emitter_(emitter) {} private: void HandleTranslationUnit(clang::ASTContext& context) override; - absl::Status GenerateAndSaveHeader(); - std::string in_file_; - const GeneratorOptions* options_; - GeneratorASTVisitor visitor_; + + Emitter& emitter_; }; class GeneratorAction : public clang::ASTFrontendAction { public: - explicit GeneratorAction(const GeneratorOptions* options) - : options_(options) {} + GeneratorAction(Emitter& emitter, const GeneratorOptions& options) + : emitter_(emitter), options_(options) {} private: std::unique_ptr CreateASTConsumer( clang::CompilerInstance&, llvm::StringRef in_file) override { return absl::make_unique(std::string(in_file), - options_); + emitter_, options_); } bool hasCodeCompletionSupport() const override { return false; } - const GeneratorOptions* options_; + Emitter& emitter_; + const GeneratorOptions& options_; }; class GeneratorFactory : public clang::tooling::FrontendActionFactory { public: - explicit GeneratorFactory(GeneratorOptions options = {}) - : options_(std::move(options)) {} + // Does not take ownership + GeneratorFactory(Emitter& emitter, const GeneratorOptions& options) + : emitter_(emitter), options_(options) {} private: #if LLVM_VERSION_MAJOR >= 10 std::unique_ptr create() override { - return absl::make_unique(&options_); + return absl::make_unique(emitter_, options_); } #else clang::FrontendAction* create() override { - return new GeneratorAction(&options_); + return new GeneratorAction(emitter_, options_); } #endif - GeneratorOptions options_; + Emitter& emitter_; + const GeneratorOptions& options_; }; +std::string GetOutputFilename(absl::string_view source_file); + } // namespace sapi #endif // SANDBOXED_API_TOOLS_CLANG_GENERATOR_GENERATOR_H_ diff --git a/sandboxed_api/tools/clang_generator/generator_tool.cc b/sandboxed_api/tools/clang_generator/generator_tool.cc index 5d5caab..43db172 100644 --- a/sandboxed_api/tools/clang_generator/generator_tool.cc +++ b/sandboxed_api/tools/clang_generator/generator_tool.cc @@ -12,16 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_split.h" #include "clang/AST/ASTContext.h" #include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/ArgumentsAdjusters.h" #include "clang/Tooling/CommonOptionsParser.h" #include "llvm/Support/CommandLine.h" +#include "sandboxed_api/sandbox2/util/file_helpers.h" #include "sandboxed_api/sandbox2/util/fileops.h" #include "sandboxed_api/tools/clang_generator/generator.h" +#include "sandboxed_api/util/status_macros.h" namespace sapi { namespace { @@ -49,11 +56,11 @@ static auto* g_sapi_functions = new llvm::cl::list( llvm::cl::cat(*g_tool_category)); static auto* g_sapi_in = new llvm::cl::list( "sapi_in", llvm::cl::CommaSeparated, - llvm::cl::desc("List of input files to analyze. Deprecated, use positional " - "arguments instead."), + llvm::cl::desc("List of input files to analyze."), llvm::cl::cat(*g_tool_category)); static auto* g_sapi_isystem = new llvm::cl::opt( - "sapi_isystem", llvm::cl::desc("Extra system include paths"), + "sapi_isystem", + llvm::cl::desc("Parameter file with extra system include paths"), llvm::cl::cat(*g_tool_category)); static auto* g_sapi_limit_scan_depth = new llvm::cl::opt( "sapi_limit_scan_depth", @@ -69,28 +76,28 @@ static auto* g_sapi_ns = new llvm::cl::opt( static auto* g_sapi_out = new llvm::cl::opt( "sapi_out", llvm::cl::desc( - "Ouput path of the generated header. If empty, simply appends .sapi.h " + "Output path of the generated header. If empty, simply appends .sapi.h " "to the basename of the first source file specified."), llvm::cl::cat(*g_tool_category)); } // namespace -GeneratorOptions GeneratorOptionsFromFlags() { +GeneratorOptions GeneratorOptionsFromFlags( + const std::vector& sources) { GeneratorOptions options; options.function_names.insert(g_sapi_functions->begin(), g_sapi_functions->end()); options.work_dir = sandbox2::file_util::fileops::GetCWD(); options.name = *g_sapi_name; options.namespace_name = *g_sapi_ns; - options.out_file = *g_sapi_out; + options.out_file = + !g_sapi_out->empty() ? *g_sapi_out : GetOutputFilename(sources.front()); options.embed_dir = *g_sapi_embed_dir; options.embed_name = *g_sapi_embed_name; return options; } -} // namespace sapi - -int main(int argc, const char** argv) { +absl::Status GeneratorMain(int argc, const char** argv) { clang::tooling::CommonOptionsParser opt_parser( argc, argv, *sapi::g_tool_category, llvm::cl::ZeroOrMore, "Generates a Sandboxed API header for C/C++ translation units."); @@ -98,9 +105,45 @@ int main(int argc, const char** argv) { for (const auto& sapi_in : *sapi::g_sapi_in) { sources.push_back(sapi_in); } + if (sources.empty()) { + return absl::InvalidArgumentError("error: no input files"); + } + + auto options = sapi::GeneratorOptionsFromFlags(sources); + sapi::Emitter emitter; clang::tooling::ClangTool tool(opt_parser.getCompilations(), sources); - return tool.run(absl::make_unique( - sapi::GeneratorOptionsFromFlags()) - .get()); + if (!sapi::g_sapi_isystem->empty()) { + std::string isystem_lines; + SAPI_RETURN_IF_ERROR(sandbox2::file::GetContents( + *sapi::g_sapi_isystem, &isystem_lines, sandbox2::file::Defaults())); + std::vector isystem = + absl::StrSplit(isystem_lines, '\n', absl::SkipWhitespace()); + for (std::string& line : isystem) { + line.insert(0, "-isystem"); + } + tool.appendArgumentsAdjuster(clang::tooling::getInsertArgumentAdjuster( + isystem, clang::tooling::ArgumentInsertPosition::BEGIN)); + } + if (int result = tool.run( + absl::make_unique(emitter, options).get()); + result != 0) { + return absl::UnknownError("header generation failed"); + } + + SAPI_ASSIGN_OR_RETURN(std::string header, emitter.EmitHeader(options)); + + SAPI_RETURN_IF_ERROR(sandbox2::file::SetContents(options.out_file, header, + sandbox2::file::Defaults())); + return absl::OkStatus(); +} + +} // namespace sapi + +int main(int argc, const char** argv) { + if (absl::Status status = sapi::GeneratorMain(argc, argv); !status.ok()) { + absl::FPrintF(stderr, "error: %s\n", status.message()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; } diff --git a/sandboxed_api/tools/clang_generator/types.cc b/sandboxed_api/tools/clang_generator/types.cc index 8efb4ef..4e12746 100644 --- a/sandboxed_api/tools/clang_generator/types.cc +++ b/sandboxed_api/tools/clang_generator/types.cc @@ -30,9 +30,9 @@ bool IsFunctionReferenceType(clang::QualType qual) { } // namespace -void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) { +void CollectRelatedTypes(clang::QualType qual, QualTypeSet* types) { if (const auto* typedef_type = qual->getAs()) { - GatherRelatedTypes(typedef_type->getDecl()->getUnderlyingType(), types); + CollectRelatedTypes(typedef_type->getDecl()->getUnderlyingType(), types); types->insert(qual); return; } @@ -43,9 +43,9 @@ void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) { ->getAs()) { // Note: Do not add the function type itself, as this will always be a // pointer argument. We only need to collect all its related types. - GatherRelatedTypes(function_type->getReturnType(), types); + CollectRelatedTypes(function_type->getReturnType(), types); for (const clang::QualType& param : function_type->getParamTypes()) { - GatherRelatedTypes(param, types); + CollectRelatedTypes(param, types); } return; } @@ -56,13 +56,13 @@ void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) { while (IsPointerOrReference(pointee)) { pointee = pointee->getPointeeType(); } - GatherRelatedTypes(pointee, types); + CollectRelatedTypes(pointee, types); return; } // C array with specified constant size (i.e. int a[42])? if (const clang::ArrayType* array_type = qual->getAsArrayTypeUnsafe()) { - GatherRelatedTypes(array_type->getElementType(), types); + CollectRelatedTypes(array_type->getElementType(), types); return; } @@ -71,7 +71,7 @@ void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) { // Collect the underlying integer type of enum classes as well, as it may // be a typedef. if (const clang::EnumDecl* decl = enum_type->getDecl(); decl->isFixed()) { - GatherRelatedTypes(decl->getIntegerType(), types); + CollectRelatedTypes(decl->getIntegerType(), types); } } types->insert(qual); @@ -81,7 +81,7 @@ void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) { if (const auto* record_type = qual->getAs()) { const clang::RecordDecl* decl = record_type->getDecl(); for (const clang::FieldDecl* field : decl->fields()) { - GatherRelatedTypes(field->getType(), types); + CollectRelatedTypes(field->getType(), types); } types->insert(qual); return; diff --git a/sandboxed_api/tools/clang_generator/types.h b/sandboxed_api/tools/clang_generator/types.h index 7947df3..78c7706 100644 --- a/sandboxed_api/tools/clang_generator/types.h +++ b/sandboxed_api/tools/clang_generator/types.h @@ -54,7 +54,7 @@ inline bool IsPointerOrReference(clang::QualType qual) { // int // SubStruct // bool -void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types); +void CollectRelatedTypes(clang::QualType qual, QualTypeSet* types); // Maps a qualified type to a fully qualified SAPI-compatible type name. This // is used for the generated code that invokes the actual function call RPC. diff --git a/sandboxed_api/tools/generator2/code.py b/sandboxed_api/tools/generator2/code.py index f483d5e..c385bfb 100644 --- a/sandboxed_api/tools/generator2/code.py +++ b/sandboxed_api/tools/generator2/code.py @@ -580,12 +580,11 @@ class _TranslationUnit(object): for i, cursor in enumerate(self._walk_preorder()): # Workaround for issue#32 + # ignore all the cursors with kinds not implemented in python bindings try: cursor.kind except ValueError: - if cursor._kind_id >= 440: # pylint: disable=protected-access - continue - raise + continue # naive way to order types: they should be ordered when walking the tree if cursor.kind.is_declaration(): self.order[cursor.hash] = i @@ -911,7 +910,7 @@ class Generator(object): result.append('#include "sandboxed_api/sandbox.h"') result.append('#include "sandboxed_api/vars.h"') - if (embed_dir is not None) and (embed_name is not None): + if embed_dir and embed_name: result.append( Generator.EMBED_INCLUDE.format( os.path.join(embed_dir, embed_name) + '_embed.h')) @@ -928,7 +927,7 @@ class Generator(object): result.append('') - if (embed_dir is not None) and (embed_name is not None): + if embed_dir and embed_name: result.append( Generator.EMBED_CLASS.format(name, embed_name.replace('-', '_'))) diff --git a/sandboxed_api/util/BUILD.bazel b/sandboxed_api/util/BUILD.bazel index d0f453c..84c0c97 100644 --- a/sandboxed_api/util/BUILD.bazel +++ b/sandboxed_api/util/BUILD.bazel @@ -46,6 +46,7 @@ cc_library( name = "statusor", hdrs = ["statusor.h"], deprecation = "Migrate to `absl::StatusOr`", + visibility = ["//visibility:public"], deps = [ "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/status:statusor", diff --git a/sandboxed_api/var_array.h b/sandboxed_api/var_array.h index 60968ba..fcf2da9 100644 --- a/sandboxed_api/var_array.h +++ b/sandboxed_api/var_array.h @@ -15,9 +15,11 @@ #ifndef SANDBOXED_API_VAR_ARRAY_H_ #define SANDBOXED_API_VAR_ARRAY_H_ +#include #include #include +#include #include "absl/base/macros.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -39,19 +41,19 @@ class Array : public Var, public Pointable { nelem_(nelem), total_size_(nelem_ * sizeof(T)), buffer_owned_(false) { - SetLocal(const_cast(reinterpret_cast(arr_))); + SetLocal(const_cast*>(arr_)); } // The array is allocated and owned by this object. explicit Array(size_t nelem) - : arr_(static_cast(malloc(sizeof(T) * nelem))), - nelem_(nelem), - total_size_(nelem_ * sizeof(T)), - buffer_owned_(true) { - SetLocal(const_cast(reinterpret_cast(arr_))); + : nelem_(nelem), total_size_(nelem_ * sizeof(T)), buffer_owned_(true) { + void* storage = malloc(sizeof(T) * nelem); + CHECK(storage != nullptr); + SetLocal(storage); + arr_ = static_cast(storage); } virtual ~Array() { if (buffer_owned_) { - free(const_cast(reinterpret_cast(arr_))); + free(const_cast*>(arr_)); } } @@ -121,7 +123,7 @@ class Array : public Var, public Pointable { arr_ = static_cast(new_addr); total_size_ = size; nelem_ = size / sizeof(T); - SetLocal(arr_); + SetLocal(new_addr); return absl::OkStatus(); } @@ -142,7 +144,7 @@ class Array : public Var, public Pointable { class CStr : public Array { public: explicit CStr(char* cstr) : Array(strlen(cstr) + 1) { - strcpy(this->GetData(), cstr); // NOLINT + std::copy(cstr, cstr + GetNElem(), GetData()); } std::string ToString() const final { @@ -159,10 +161,7 @@ class ConstCStr : public Array { : Array(cstr, strlen(cstr) + 1) {} std::string ToString() const final { - if (GetData() == nullptr) { - return "CStr: [nullptr]"; - } - return absl::StrCat("CStr: len(w/o NUL):", strlen(GetData()), ", ['", + return absl::StrCat("ConstCStr: len(w/o NUL):", strlen(GetData()), ", ['", GetData(), "']"); } };