Merge branch 'master' into jsonnet

.bazelignore

@@ -0,0 +1,4 @@
+# Using CMake or own WORKSPACE
+oss-internship-2020
+# Uses its own WORKSPACE
+sandboxed_api/examples/hello_sapi

.gitmodules

@@ -7,3 +7,9 @@
 [submodule "oss-internship-2020/pffft/master"]
 	path = oss-internship-2020/pffft/master
 	url = https://bitbucket.org/jpommier/pffft/src/master/
+[submodule "oss-internship-2020/curl/curl_wrapper/curl"]
+	path = oss-internship-2020/curl/curl_wrapper/curl
+	url = https://github.com/curl/curl
+[submodule "oss-internship-2020/gdal/gdal"]
+	path = oss-internship-2020/gdal/gdal
+	url = https://github.com/OSGeo/gdal/tree/master/gdal

CMakeLists.txt

@@ -14,6 +14,16 @@
 
 cmake_minimum_required(VERSION 3.12)
 
+# Fix Ninja generator output to not rebuild entire sub-trees needlessly.
+if(CMAKE_GENERATOR MATCHES "Ninja")
+  file(WRITE "${CMAKE_BINARY_DIR}/UserMakeRulesOverride.cmake"
+    "string(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_C \"\${CMAKE_DEPFILE_FLAGS_C}\")\n"
+    "string(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_CXX \"\${CMAKE_DEPFILE_FLAGS_CXX}\")\n"
+  )
+  set(CMAKE_USER_MAKE_RULES_OVERRIDE
+      "${CMAKE_BINARY_DIR}/UserMakeRulesOverride.cmake" CACHE INTERNAL "")
+endif()
+
 project(SandboxedAPI C CXX ASM)
 
 # SAPI-wide setting for the language level
@@ -31,15 +41,11 @@ include(SapiDeps)
 include(SapiUtil)
 include(SapiBuildDefs)
 
-# Fix Ninja generator output to not rebuild entire sub-trees needlessly.
+# Allow the header generator to auto-configure include paths
-if(CMAKE_GENERATOR MATCHES "Ninja")
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-  file(WRITE "${SAPI_BINARY_DIR}/UserMakeRulesOverride.cmake"
+
-    "STRING(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_C \"\${CMAKE_DEPFILE_FLAGS_C}\")\n"
+# Allow the header generator to auto-configure include paths
-    "STRING(REPLACE \"-MD\" \"-MMD\" CMAKE_DEPFILE_FLAGS_CXX \"\${CMAKE_DEPFILE_FLAGS_CXX}\")\n"
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-  )
-  set(CMAKE_USER_MAKE_RULES_OVERRIDE
-      "${SAPI_BINARY_DIR}/UserMakeRulesOverride.cmake" CACHE INTERNAL "")
-endif()
 
 if (SAPI_FORCE_COLOR_OUTPUT)
   if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")  # GCC

cmake/SapiBuildDefs.cmake

@@ -123,7 +123,7 @@ function(add_sapi_library)
   endif()
 
   # Interface
-  list_join(_sapi_FUNCTIONS "," _sapi_funcs)
+  list(JOIN _sapi_FUNCTIONS "," _sapi_funcs)
   foreach(src IN LISTS _sapi_INPUTS)
     get_filename_component(src "${src}" ABSOLUTE)
     list(APPEND _sapi_full_inputs "${src}")
@@ -132,44 +132,44 @@ function(add_sapi_library)
     set(_sapi_embed_dir "${CMAKE_CURRENT_BINARY_DIR}")
     set(_sapi_embed_name "${_sapi_NAME}")
   endif()
+
+  set(_sapi_isystem "${_sapi_NAME}.isystem}")
+  list(APPEND _sapi_generator_args
+    "--sapi_name=${_sapi_LIBRARY_NAME}"
+    "--sapi_out=${_sapi_gen_header}"
+    "--sapi_embed_dir=${_sapi_embed_dir}"
+    "--sapi_embed_name=${_sapi_embed_name}"
+    "--sapi_functions=${_sapi_funcs}"
+    "--sapi_ns=${_sapi_NAMESPACE}"
+    "--sapi_isystem=${_sapi_isystem}"
+  )
+  list(JOIN _sapi_full_inputs "," _sapi_full_inputs)
   if(SAPI_ENABLE_GENERATOR)
-    add_custom_command(
+    list(APPEND _sapi_generator_command
-      OUTPUT "${_sapi_gen_header}"
+      sapi_generator_tool
-      COMMAND sapi_generator_tool
+      -p "${CMAKE_CURRENT_BINARY_DIR}"
-              "--sapi_name=${_sapi_LIBRARY_NAME}"
+      ${_sapi_generator_args}
-              "--sapi_out=${_sapi_gen_header}"
+      ${_sapi_full_inputs}
-              "--sapi_embed_dir=${_sapi_embed_dir}"
-              "--sapi_embed_name=${_sapi_embed_name}"
-              "--sapi_functions=${_sapi_funcs}"
-              "--sapi_ns=${_sapi_NAMESPACE}"
-              ${_sapi_full_inputs}
-      COMMENT "Generating interface"
-      DEPENDS ${_sapi_INPUTS}
-      VERBATIM
     )
   else()
-    set(_sapi_isystem "${_sapi_NAME}.isystem")
+    list(APPEND _sapi_generator_command
-    list_join(_sapi_full_inputs "," _sapi_full_inputs)
+      "${SAPI_PYTHON3_EXECUTABLE}" -B
-    add_custom_command(
+      "${SAPI_SOURCE_DIR}/sandboxed_api/tools/generator2/sapi_generator.py"
-      OUTPUT "${_sapi_gen_header}" "${_sapi_isystem}"
+      ${_sapi_generator_args}
-      COMMAND sh -c
+      "--sapi_in=${_sapi_full_inputs}"
-              "${CMAKE_CXX_COMPILER} -E -x c++ -v /dev/null 2>&1 | \
-               awk '/> search starts here:/{f=1;next}/^End of search/{f=0}f{print $1}' \
-               > \"${_sapi_isystem}\""
-      COMMAND "${SAPI_PYTHON3_EXECUTABLE}" -B
-              "${SAPI_SOURCE_DIR}/sandboxed_api/tools/generator2/sapi_generator.py"
-              "--sapi_name=${_sapi_LIBRARY_NAME}"
-              "--sapi_out=${_sapi_gen_header}"
-              "--sapi_embed_dir=${_sapi_embed_dir}"
-              "--sapi_embed_name=${_sapi_embed_name}"
-              "--sapi_functions=${_sapi_funcs}"
-              "--sapi_ns=${_sapi_NAMESPACE}"
-              "--sapi_isystem=${_sapi_isystem}"
-              "--sapi_in=${_sapi_full_inputs}"
-      COMMENT "Generating interface"
-      VERBATIM
     )
   endif()
+  add_custom_command(
+    OUTPUT "${_sapi_gen_header}" "${_sapi_isystem}"
+    COMMAND sh -c
+            "${CMAKE_CXX_COMPILER} -E -x c++ -v /dev/null 2>&1 | \
+             awk '/> search starts here:/{f=1;next}/^End of search/{f=0}f{print $1}' \
+             > \"${_sapi_isystem}\""
+    COMMAND ${_sapi_generator_command}
+    COMMENT "Generating interface"
+    DEPENDS ${_sapi_INPUTS}
+    VERBATIM
+  )
 
   # Library with the interface
   if(NOT _sapi_SOURCES)

cmake/SapiUtil.cmake

@@ -35,16 +35,6 @@ function(create_directory_symlink SOURCE DESTINATION)
   endif()
 endfunction()
 
-# Implements list(JOIN ...) for CMake < 3.12.
-function(list_join LIST SEP OUTPUT)
-  foreach(item IN LISTS ${LIST})
-    set(_concat "${_concat}${SEP}${item}")
-  endforeach()
-  string(LENGTH "${SEP}" _len)
-  string(SUBSTRING "${_concat}" ${_len} -1 _concat)
-  set(${OUTPUT} "${_concat}" PARENT_SCOPE)
-endfunction()
-
 # Helper function that behaves just like Protobuf's protobuf_generate_cpp(),
 # except that it strips import paths. This is necessary, because CMake's
 # protobuf rules don't work well with imports across different directories.

oss-internship-2020/curl/.gitignore

@@ -0,0 +1 @@
+build/

oss-internship-2020/curl/CMakeLists.txt

@@ -0,0 +1,144 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.16)
+
+project(libcurl_sandbox)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+option(CURL_SAPI_ENABLE_EXAMPLES "" ON)
+option(CURL_SAPI_ENABLE_TESTS "" ON)
+
+# Add callbacks used by examples and tests
+if (CURL_SAPI_ENABLE_EXAMPLES OR CURL_SAPI_ENABLE_TESTS)
+  list(APPEND CURL_SAPI_CALLBACKS
+    "${CMAKE_CURRENT_SOURCE_DIR}/callbacks/callbacks.h"
+    "${CMAKE_CURRENT_SOURCE_DIR}/callbacks/callbacks.cc"
+  )
+endif()
+
+# Add folder containing the non-sandboxed custom curl library
+add_subdirectory(curl_wrapper)
+
+# Setup Sandboxed API
+set(SAPI_ROOT "" CACHE PATH "Path to the Sandboxed API source tree")
+set(SAPI_ENABLE_EXAMPLES ${CURL_SAPI_ENABLE_EXAMPLES} CACHE BOOL "" FORCE)
+set(SAPI_ENABLE_TESTS ${CURL_SAPI_ENABLE_TESTS} CACHE BOOL "" FORCE)
+add_subdirectory(
+  "${SAPI_ROOT}"
+  "${CMAKE_BINARY_DIR}/sandboxed-api-build"
+  EXCLUDE_FROM_ALL
+)
+
+# Generate SAPI header
+add_sapi_library(curl_sapi
+
+  # List of all the methods in https://curl.haxx.se/libcurl/c/allfuncs.html
+  # Some are added or modified because the original ones are not supported
+  # by Sandboxed API (details can be found in curl_wrapper.h)
+  FUNCTIONS curl_easy_cleanup
+            curl_easy_duphandle
+            curl_easy_escape
+            curl_easy_getinfo
+            curl_easy_getinfo_ptr
+            curl_easy_init
+            curl_easy_pause
+            curl_easy_perform
+            curl_easy_recv
+            curl_easy_reset
+            curl_easy_send
+            curl_easy_setopt
+            curl_easy_setopt_ptr
+            curl_easy_setopt_long
+            curl_easy_setopt_curl_off_t
+            curl_easy_strerror
+            curl_easy_unescape
+            curl_easy_upkeep
+            curl_free
+            curl_getdate_sapi
+            curl_global_cleanup
+            curl_global_init
+            curl_global_init_mem
+            curl_global_sslset
+            curl_mime_addpart
+            curl_mime_data
+            curl_mime_data_cb
+            curl_mime_encoder
+            curl_mime_filedata
+            curl_mime_filename
+            curl_mime_free
+            curl_mime_headers
+            curl_mime_init
+            curl_mime_name
+            curl_mime_subparts
+            curl_mime_type
+            curl_multi_add_handle
+            curl_multi_assign
+            curl_multi_cleanup
+            curl_multi_fdset_sapi
+            curl_multi_info_read
+            curl_multi_init
+            curl_multi_perform
+            curl_multi_remove_handle
+            curl_multi_setopt
+            curl_multi_setopt_ptr
+            curl_multi_setopt_long
+            curl_multi_setopt_curl_off_t
+            curl_multi_socket_action
+            curl_multi_strerror
+            curl_multi_timeout
+            curl_multi_poll_sapi
+            curl_multi_wait_sapi
+            curl_multi_wakeup
+            curl_share_init
+            curl_share_setopt
+            curl_share_setopt_ptr
+            curl_share_setopt_long
+            curl_share_strerror
+            curl_slist_append
+            curl_slist_free_all
+            curl_url
+            curl_url_cleanup
+            curl_url_dup
+            curl_url_get
+            curl_url_set
+            curl_version
+            curl_version_info
+
+  INPUTS curl_wrapper/curl/include/curl/curl.h
+         curl_wrapper/curl_wrapper.h
+
+  LIBRARY curl_wrapper_and_callbacks
+
+  LIBRARY_NAME Curl
+
+  NAMESPACE ""
+)
+
+# Include generated SAPI header
+target_include_directories(curl_sapi INTERFACE
+  "${PROJECT_BINARY_DIR}"
+)
+
+# Add examples
+if (CURL_SAPI_ENABLE_EXAMPLES)
+  add_subdirectory(examples)
+endif()
+
+# Add tests
+if (CURL_SAPI_ENABLE_TESTS)
+  add_subdirectory(tests)
+endif()

oss-internship-2020/curl/README.md

@@ -0,0 +1,112 @@
+# LibCurl Sandbox
+
+This library is a sandboxed version of curl's C API,
+[libcurl](https://curl.haxx.se/libcurl/c/), implemented using Sandboxed API.
+
+## Setup
+
+The repository can be cloned using: `git clone --recursive [URL to this repo]`
+The `--recursive` flag ensures that submodules are also cloned.
+
+Alternatively, if the repository has already been cloned but the submodules have
+not, these can be cloned using: `git submodule update --init --recursive`
+
+The full list of Sandboxed API dependencies can be found on
+[Sandboxed API Getting Started page](https://developers.google.com/sandboxed-api/docs/getting-started).
+
+The following commands, used from the current `curl/` directory, build the
+library: `mkdir -p build cd build cmake .. -G Ninja -D SAPI_ROOT=[path to
+sandboxed-api] cmake --build .`
+
+## Implementation details
+
+All of libcurl's methods are supported by the library. However, a few of these
+have different signatures defined in the sandboxed header `custom_curl.h`, which
+wraps and extends libcurl.
+
+This is necessary because Sandboxed API sandboxes most of libcurl correctly, but
+encounters some issues when sandboxing a few methods. The simplest solution is
+wrapping these methods into wrapper methods that accomplish the same tasks but
+can also be sandboxed.
+
+The next sections describe the issues encountered and contain some information
+on the signatures of the wrapper methods solving these issues.
+
+#### Variadic methods
+
+Variadic methods are currently not supported by Sandboxed API. To solve this,
+these methods are defined with an additional explicit parameter in
+`custom_curl.h`.
+
+The methods are: - `curl_easy_setopt`. Use `curl_easy_setopt_ptr`,
+`curl_easy_setopt_long` or `curl_easy_setopt_curl_off_t` instead. -
+`curl_easy_getinfo`. Use `curl_easy_getinfo_ptr` instead. - `curl_multi_setopt`.
+Use `curl_multi_setopt_ptr`, `curl_multi_setopt_long` or
+`curl_multi_setopt_curl_off_t` instead. - `curl_share_setopt`. Use
+`curl_share_setopt_ptr` or `curl_share_setopt_long` instead
+
+#### Methods with incomplete array arguments
+
+Incomplete array arguments are currently not supported by Sandboxed API. To
+solve this, methods taking an incomplete array argument have a wrapper in
+`custom_curl.h`, and take a pointer as the argument.
+
+The methods are: - `curl_multi_poll`. Use `curl_multi_poll_sapi` instead. -
+`curl_multi_wait`. Use `curl_multi_wait_sapi` instead.
+
+#### Methods with conflicts on the generated header
+
+Some methods create conflicts on the generated header because of redefined
+`#define` directives from files included by the header. To solve this, the
+conflicting types and methods are redefined in `custom_curl.h`.
+
+The types are: - `time_t`. Use `time_t_sapi` instead. - `fd_set`. Use
+`fd_set_sapi` instead.
+
+The methods are: - `curl_getdate`. Use `curl_getdate_sapi` instead. -
+`curl_multi_fdset`. Use `curl_multi_fdset_sapi` instead.
+
+#### Function pointers
+
+The functions whose pointers will be passed to the library's methods
+(*callbacks*) can't be implemented in the files making use of the library, but
+must be in other files. These files must be compiled together with the library,
+and this is done by adding their absolute path to the cmake variable
+`CURL_SAPI_CALLBACKS`.
+
+The pointers can then be obtained using an `RPCChannel` object, as shown in
+`example2.cc`.
+
+## Examples
+
+The `examples` directory contains the sandboxed versions of example source codes
+taken from [this page](https://curl.haxx.se/libcurl/c/example.html) on curl's
+website. More information about each example can be found in the examples'
+[README](examples/README.md).
+
+To build these examples when building the library, the cmake variable
+`CURL_SAPI_ENABLE_EXAMPLES` must be set to `ON`. This enables Sandboxed API
+examples as well.
+
+## Policy
+
+The `sandbox.h` file contains a policy allowing all is necessary for libcurl to
+perform simple requests. It is used by all the examples, except by example3.
+This example needs some additional policies and files in its namespace (since it
+uses HTTPS), and the file `example3.cc` shows how to easily extend an existing
+policy.
+
+## Testing
+
+The `tests` folder contains some test cases created using Google Test. The class
+`CurlTestUtils` is used to facilitate some tasks that all test cases need,
+including the setup of a mock local server on which test requests are performed.
+
+To build these tests when building the library, the cmake variable
+`CURL_SAPI_ENABLE_TESTS` must be set to `ON`. This enables Sandboxed API tests
+as well.
+
+## Callbacks
+
+The `callbacks.h` and `callbacks.cc` files implement all the callbacks used by
+examples and tests.

oss-internship-2020/curl/callbacks/callbacks.cc

@@ -0,0 +1,37 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "callbacks.h"  // NOLINT(build/include)
+
+#include <cstdlib>
+#include <cstring>
+
+#include "sandboxed_api/vars.h"
+
+size_t WriteToMemory(char* contents, size_t size, size_t num_bytes,
+                     void* userp) {
+  size_t real_size = size * num_bytes;
+  auto* mem = static_cast<sapi::LenValStruct*>(userp);
+
+  char* ptr = static_cast<char*>(realloc(mem->data, mem->size + real_size + 1));
+  if (ptr == nullptr) return 0;
+
+  mem->data = ptr;
+  auto data = static_cast<char*>(mem->data);
+  memcpy(&(data[mem->size]), contents, real_size);
+  mem->size += real_size;
+  data[mem->size] = 0;
+
+  return real_size;
+}

oss-internship-2020/curl/callbacks/callbacks.h

@@ -0,0 +1,24 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TESTS_CALLBACKS_H
+#define TESTS_CALLBACKS_H
+
+#include <curl/curl.h>
+
+// Append contents to the string stored by userp, which is a sapi::LenValStruct*
+extern "C" size_t WriteToMemory(char* contents, size_t size, size_t num_bytes,
+                                void* userp);
+
+#endif  // TESTS_CALLBACKS_H

oss-internship-2020/curl/curl_wrapper/CMakeLists.txt

@@ -0,0 +1,37 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Wrapper library including curl, wrappers for some methods and callbacks
+# The CURL_SAPI_CALLBACKS variable should contain the absolute paths of
+# all the files implementing the callbacks
+add_library(curl_wrapper_and_callbacks OBJECT
+  curl_wrapper.h
+  curl_wrapper.cc
+  "${CURL_SAPI_CALLBACKS}"
+)
+set_target_properties(curl_wrapper_and_callbacks
+  PROPERTIES LINKER_LANGUAGE C
+)
+
+# Flags needed to build curl statically
+set(CURL_HIDDEN_SYMBOLS OFF)
+set(BUILD_SHARED_LIBS OFF)
+
+# Link the wrapper to the original curl library (testing is disabled in curl)
+set(BUILD_TESTING OFF)
+add_subdirectory(curl)
+target_link_libraries(curl_wrapper_and_callbacks
+  CURL::libcurl
+  sapi::sapi
+)

oss-internship-2020/curl/curl_wrapper/curl_wrapper.cc

@@ -0,0 +1,86 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "curl_wrapper.h"  // NOLINT(build/include)
+
+CURLcode curl_easy_setopt_ptr(CURL* handle, CURLoption option,
+                              void* parameter) {
+  return curl_easy_setopt(handle, option, parameter);
+}
+
+CURLcode curl_easy_setopt_long(CURL* handle, CURLoption option,
+                               long parameter) {
+  return curl_easy_setopt(handle, option, parameter);
+}
+
+CURLcode curl_easy_setopt_curl_off_t(CURL* handle, CURLoption option,
+                                     curl_off_t parameter) {
+  return curl_easy_setopt(handle, option, parameter);
+}
+
+CURLcode curl_easy_getinfo_ptr(CURL* handle, CURLINFO option, void* parameter) {
+  return curl_easy_getinfo(handle, option, parameter);
+}
+
+time_t_sapi curl_getdate_sapi(char* datestring, time_t_sapi* now) {
+  return curl_getdate(datestring, now);
+}
+
+CURLMcode curl_multi_fdset_sapi(CURLM* multi_handle, fd_set_sapi* read_fd_set,
+                                fd_set_sapi* write_fd_set,
+                                fd_set_sapi* exc_fd_set, int* max_fd) {
+  return curl_multi_fdset(multi_handle, read_fd_set, write_fd_set, exc_fd_set,
+                          max_fd);
+}
+
+CURLMcode curl_multi_setopt_ptr(CURLM* handle, CURLMoption option,
+                                void* parameter) {
+  return curl_multi_setopt(handle, option, parameter);
+}
+
+CURLMcode curl_multi_setopt_long(CURLM* handle, CURLMoption option,
+                                 long parameter) {
+  return curl_multi_setopt(handle, option, parameter);
+}
+
+CURLMcode curl_multi_setopt_curl_off_t(CURLM* handle, CURLMoption option,
+                                       curl_off_t parameter) {
+  return curl_multi_setopt(handle, option, parameter);
+}
+
+CURLMcode curl_multi_poll_sapi(CURLM* multi_handle,
+                               struct curl_waitfd* extra_fds,
+                               unsigned int extra_nfds, int timeout_ms,
+                               int* numfds) {
+  return curl_multi_poll(multi_handle, extra_fds, extra_nfds, timeout_ms,
+                         numfds);
+}
+
+CURLMcode curl_multi_wait_sapi(CURLM* multi_handle,
+                               struct curl_waitfd* extra_fds,
+                               unsigned int extra_nfds, int timeout_ms,
+                               int* numfds) {
+  return curl_multi_wait(multi_handle, extra_fds, extra_nfds, timeout_ms,
+                         numfds);
+}
+
+CURLSHcode curl_share_setopt_ptr(CURLSH* handle, CURLSHoption option,
+                                 void* parameter) {
+  return curl_share_setopt(handle, option, parameter);
+}
+
+CURLSHcode curl_share_setopt_long(CURLSH* handle, CURLSHoption option,
+                                  long parameter) {
+  return curl_share_setopt(handle, option, parameter);
+}

oss-internship-2020/curl/curl_wrapper/curl_wrapper.h

@@ -0,0 +1,82 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Wrapper for curl library
+
+#ifndef CURL_WRAPPER_H
+#define CURL_WRAPPER_H
+
+#include <curl/curl.h>
+
+extern "C" {
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLcode curl_easy_setopt_ptr(CURL* handle, CURLoption option, void* parameter);
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLcode curl_easy_setopt_long(CURL* handle, CURLoption option, long parameter);
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLcode curl_easy_setopt_curl_off_t(CURL* handle, CURLoption option,
+                                     curl_off_t parameter);
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLcode curl_easy_getinfo_ptr(CURL* handle, CURLINFO option, void* parameter);
+
+// The typedef and wrapper method are needed because the original method has
+// some conflicts in curl_sapi.sapi.h
+typedef time_t time_t_sapi;
+time_t_sapi curl_getdate_sapi(char* datestring, time_t_sapi* now);
+
+// The typedef and wrapper method are needed because the original method has
+// some conflicts in curl_sapi.sapi.h
+typedef fd_set fd_set_sapi;
+CURLMcode curl_multi_fdset_sapi(CURLM* multi_handle, fd_set_sapi* read_fd_set,
+                                fd_set_sapi* write_fd_set,
+                                fd_set_sapi* exc_fd_set, int* max_fd);
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLMcode curl_multi_setopt_ptr(CURLM* handle, CURLMoption option,
+                                void* parameter);
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLMcode curl_multi_setopt_long(CURLM* handle, CURLMoption option,
+                                 long parameter);
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLMcode curl_multi_setopt_curl_off_t(CURLM* handle, CURLMoption option,
+                                       curl_off_t parameter);
+
+// The wrapper method is needed because incomplete array type is not supported
+CURLMcode curl_multi_poll_sapi(CURLM* multi_handle,
+                               struct curl_waitfd* extra_fds,
+                               unsigned int extra_nfds, int timeout_ms,
+                               int* numfds);
+
+// The wrapper method is needed because incomplete array type is not supported
+CURLMcode curl_multi_wait_sapi(CURLM* multi_handle,
+                               struct curl_waitfd* extra_fds,
+                               unsigned int extra_nfds, int timeout_ms,
+                               int* numfds);
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLSHcode curl_share_setopt_ptr(CURLSH* handle, CURLSHoption option,
+                                 void* parameter);
+
+// The wrapper method is needed to make the variadic argument explicit
+CURLSHcode curl_share_setopt_long(CURLSH* handle, CURLSHoption option,
+                                  long parameter);
+}
+
+#endif  // CURL_WRAPPER_H

oss-internship-2020/curl/examples/CMakeLists.txt

@@ -0,0 +1,76 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# All the examples are sandboxed versions of curl's examples
+# (https://curl.haxx.se/libcurl/c/example.html)
+
+# Example 1: simple.c
+add_executable(example1
+  example1.cc
+  ../sandbox.h
+)
+target_link_libraries(example1 PRIVATE
+  curl_sapi
+  sapi::sapi
+)
+
+# Example 2: getinmemory.c
+add_executable(example2
+  example2.cc
+  ../sandbox.h
+)
+target_link_libraries(example2 PRIVATE
+  curl_sapi
+  sapi::sapi
+)
+
+# Example 3: simplessl.c
+add_executable(example3
+  example3.cc
+  ../sandbox.h
+)
+target_link_libraries(example3 PRIVATE
+  curl_sapi
+  sapi::sapi
+)
+
+# Example 4: multi-poll.c
+add_executable(example4
+  example4.cc
+  ../sandbox.h
+)
+target_link_libraries(example4 PRIVATE
+  curl_sapi
+  sapi::sapi
+)
+
+# Example 5: multithread.c
+add_executable(example5
+  example5.cc
+  ../sandbox.h
+)
+target_link_libraries(example5 PRIVATE
+  curl_sapi
+  sapi::sapi
+)
+
+# Example 6: simple.c (using transactions)
+add_executable(example6
+  example6.cc
+  ../sandbox.h
+)
+target_link_libraries(example6 PRIVATE
+  curl_sapi
+  sapi::sapi
+)

oss-internship-2020/curl/examples/README.md

@@ -0,0 +1,38 @@
+# LibCurl Sandbox Examples
+
+Each example in this folder is the sandboxed version of a code snippet from
+[this page](https://curl.haxx.se/libcurl/c/example.html) on curl's website.
+These examples perform some basic tasks using libcurl, and can be useful both to
+understand how to use LibCurl Sandbox, but also to get an idea of how regular
+and sandboxed code compare to each other.
+
+This is the list of the examples:
+
+-   **example1**: sandboxed version of
+    [simple.c](https://curl.haxx.se/libcurl/c/simple.html). Really simple HTTP
+    request, downloads and prints out the page at
+    [example.com](http://example.com).
+-   **example2**: sandboxed version of
+    [getinmemory.c](https://curl.haxx.se/libcurl/c/getinmemory.html). Same HTTP
+    request as example1. The difference is that this example uses a callback to
+    save the page directly in memory. Only the page size is printed out.
+-   **example3**: sandboxed version of
+    [simplessl.c](https://curl.haxx.se/libcurl/c/simplessl.html). HTTPS request
+    of the [example.com](https://example.com) page, using SSL authentication.
+    This script takes 4 arguments (SSL certificates file, SSL keys file, SSL
+    keys password and CA certificates files), and prints out the page.
+-   **example4**: sandboxed version of
+    [multi-poll.c](https://curl.haxx.se/libcurl/c/multi-poll.html). Same HTTP
+    request as example1, with the addition of a polling method that can be used
+    to track the status of the request. The page is printed out after it is
+    downloaded.
+-   **example5**: sandboxed version of
+    [multithread.c](https://curl.haxx.se/libcurl/c/multithread.html). Four HTTP
+    request of the pages [example.com](http://example.com),
+    [example.edu](http://example.edu), [example.net](http://example.net) and
+    [example.org](http://example.org), performed at the same time using
+    libcurl's multithreading methods. The pages are printed out.
+-   **example6**: sandboxed version of
+    [simple.c](https://curl.haxx.se/libcurl/c/simple.html). Performs the same
+    tasks as example1, but Sandbox API Transactions are used to show how they
+    can be used to perform a simple request.

oss-internship-2020/curl/examples/example1.cc

@@ -0,0 +1,78 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Sandboxed version of simple.c
+// Simple HTTP GET request
+
+#include <cstdlib>
+
+#include "../sandbox.h"  // NOLINT(build/include)
+
+int main(int argc, char* argv[]) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  google::InitGoogleLogging(argv[0]);
+
+  absl::Status status;
+
+  // Initialize sandbox2 and sapi
+  CurlSapiSandbox sandbox;
+  status = sandbox.Init();
+  if (!status.ok()) {
+    LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status;
+  }
+  CurlApi api(&sandbox);
+
+  // Initialize the curl session
+  absl::StatusOr<CURL*> curl_handle = api.curl_easy_init();
+  if (!curl_handle.ok()) {
+    LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status();
+  }
+  sapi::v::RemotePtr curl(curl_handle.value());
+  if (!curl.GetValue()) LOG(FATAL) << "curl_easy_init failed: curl is NULL";
+
+  absl::StatusOr<int> curl_code;
+
+  // Specify URL to get
+  sapi::v::ConstCStr url("http://example.com");
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Set the library to follow a redirection
+  curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_FOLLOWLOCATION, 1l);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status();
+  }
+
+  // Disable authentication of peer certificate
+  curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_SSL_VERIFYPEER, 0l);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status();
+  }
+
+  // Perform the request
+  curl_code = api.curl_easy_perform(&curl);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status();
+  }
+
+  // Cleanup curl
+  status = api.curl_easy_cleanup(&curl);
+  if (!status.ok()) {
+    LOG(FATAL) << "curl_easy_cleanup failed: " << status;
+  }
+
+  return EXIT_SUCCESS;
+}

oss-internship-2020/curl/examples/example2.cc

@@ -0,0 +1,107 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Sandboxed version of getinmemory.c
+// HTTP GET request using callbacks
+
+#include <cstdlib>
+#include <iostream>
+
+#include "../sandbox.h"  // NOLINT(build/include)
+
+int main(int argc, char* argv[]) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  google::InitGoogleLogging(argv[0]);
+
+  absl::Status status;
+
+  // Initialize sandbox2 and sapi
+  CurlSapiSandbox sandbox;
+  status = sandbox.Init();
+  if (!status.ok()) {
+    LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status;
+  }
+  CurlApi api(&sandbox);
+
+  // Generate pointer to WriteMemoryCallback function
+  void* function_ptr;
+  status = sandbox.rpc_channel()->Symbol("WriteToMemory", &function_ptr);
+  if (!status.ok()) {
+    LOG(FATAL) << "sapi::Sandbox::rpc_channel().Symbol failed: " << status;
+  }
+  sapi::v::RemotePtr remote_function_ptr(function_ptr);
+
+  // Initialize the curl session
+  absl::StatusOr<CURL*> curl_handle = api.curl_easy_init();
+  if (!curl_handle.ok()) {
+    LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status();
+  }
+  sapi::v::RemotePtr curl(curl_handle.value());
+  if (!curl.GetValue()) {
+    LOG(FATAL) << "curl_easy_init failed: curl is NULL";
+  }
+
+  absl::StatusOr<int> curl_code;
+
+  // Specify URL to get
+  sapi::v::ConstCStr url("http://example.com");
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Set WriteMemoryCallback as the write function
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_WRITEFUNCTION,
+                                       &remote_function_ptr);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Pass 'chunk' struct to the callback function
+  sapi::v::LenVal chunk(0);
+  curl_code =
+      api.curl_easy_setopt_ptr(&curl, CURLOPT_WRITEDATA, chunk.PtrBoth());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Set a user agent
+  sapi::v::ConstCStr user_agent("libcurl-agent/1.0");
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_USERAGENT,
+                                       user_agent.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Perform the request
+  curl_code = api.curl_easy_perform(&curl);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status();
+  }
+
+  // Retrieve memory size
+  status = sandbox.TransferFromSandboxee(&chunk);
+  if (!status.ok()) {
+    LOG(FATAL) << "sandbox.TransferFromSandboxee failed: " << status;
+  }
+  std::cout << "memory size: " << chunk.GetDataSize() << " bytes" << std::endl;
+
+  // Cleanup curl
+  status = api.curl_easy_cleanup(&curl);
+  if (!status.ok()) {
+    LOG(FATAL) << "curl_easy_cleanup failed: " << status;
+  }
+
+  return EXIT_SUCCESS;
+}

oss-internship-2020/curl/examples/example3.cc

@@ -0,0 +1,166 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Sandboxed version of simplessl.c
+// HTTPS GET request
+
+#include <cstdlib>
+
+#include "../sandbox.h"  // NOLINT(build/include)
+
+class CurlSapiSandboxEx3 : public CurlSapiSandbox {
+ public:
+  CurlSapiSandboxEx3(std::string ssl_certificate, std::string ssl_key,
+                     std::string ca_certificates)
+      : ssl_certificate(ssl_certificate),
+        ssl_key(ssl_key),
+        ca_certificates(ca_certificates) {}
+
+ private:
+  std::unique_ptr<sandbox2::Policy> ModifyPolicy(
+      sandbox2::PolicyBuilder*) override {
+    // Add the syscalls and files missing in CurlSandbox to a new PolicyBuilder
+    auto policy_builder = std::make_unique<sandbox2::PolicyBuilder>();
+    (*policy_builder)
+        .AllowFutexOp(FUTEX_WAIT_PRIVATE)
+        .AllowGetPIDs()
+        .AllowGetRandom()
+        .AllowHandleSignals()
+        .AllowSyscall(__NR_sysinfo)
+        .AddFile(ssl_certificate)
+        .AddFile(ssl_key)
+        .AddFile(ca_certificates);
+    // Provide the new PolicyBuilder to ModifyPolicy in CurlSandbox
+    return CurlSapiSandbox::ModifyPolicy(policy_builder.get());
+  }
+
+  std::string ssl_certificate;
+  std::string ssl_key;
+  std::string ca_certificates;
+};
+
+int main(int argc, char* argv[]) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  google::InitGoogleLogging(argv[0]);
+
+  absl::Status status;
+
+  // Get input parameters (should be absolute paths)
+  if (argc != 5) {
+    LOG(FATAL) << "wrong number of arguments (4 expected)";
+  }
+  std::string ssl_certificate = argv[1];
+  std::string ssl_key = argv[2];
+  std::string ssl_key_password = argv[3];
+  std::string ca_certificates = argv[4];
+
+  // Initialize sandbox2 and sapi
+  CurlSapiSandboxEx3 sandbox(ssl_certificate, ssl_key, ca_certificates);
+  status = sandbox.Init();
+  if (!status.ok()) {
+    LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status;
+  }
+  CurlApi api(&sandbox);
+
+  absl::StatusOr<int> curl_code;
+
+  // Initialize curl (CURL_GLOBAL_DEFAULT = 3)
+  curl_code = api.curl_global_init(3l);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_global_init failed: " << curl_code.status();
+  }
+
+  // Initialize curl easy handle
+  absl::StatusOr<CURL*> curl_handle = api.curl_easy_init();
+  if (!curl_handle.ok()) {
+    LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status();
+  }
+  sapi::v::RemotePtr curl(curl_handle.value());
+  if (!curl.GetValue()) {
+    LOG(FATAL) << "curl_easy_init failed: curl is NULL";
+  }
+
+  // Specify URL to get (using HTTPS)
+  sapi::v::ConstCStr url("https://example.com");
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Set the SSL certificate type to "PEM"
+  sapi::v::ConstCStr ssl_cert_type("PEM");
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLCERTTYPE,
+                                       ssl_cert_type.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Set the certificate for client authentication
+  sapi::v::ConstCStr sapi_ssl_certificate(ssl_certificate.c_str());
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLCERT,
+                                       sapi_ssl_certificate.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Set the private key for client authentication
+  sapi::v::ConstCStr sapi_ssl_key(ssl_key.c_str());
+  curl_code =
+      api.curl_easy_setopt_ptr(&curl, CURLOPT_SSLKEY, sapi_ssl_key.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Set the password used to protect the private key
+  sapi::v::ConstCStr sapi_ssl_key_password(ssl_key_password.c_str());
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_KEYPASSWD,
+                                       sapi_ssl_key_password.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Set the file with the certificates vaildating the server
+  sapi::v::ConstCStr sapi_ca_certificates(ca_certificates.c_str());
+  curl_code = api.curl_easy_setopt_ptr(&curl, CURLOPT_CAINFO,
+                                       sapi_ca_certificates.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Verify the authenticity of the server
+  curl_code = api.curl_easy_setopt_long(&curl, CURLOPT_SSL_VERIFYPEER, 1L);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_long failed: " << curl_code.status();
+  }
+
+  // Perform the request
+  curl_code = api.curl_easy_perform(&curl);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status();
+  }
+
+  // Cleanup curl easy handle
+  status = api.curl_easy_cleanup(&curl);
+  if (!status.ok()) {
+    LOG(FATAL) << "curl_easy_cleanup failed: " << status;
+  }
+
+  // Cleanup curl
+  status = api.curl_global_cleanup();
+  if (!status.ok()) {
+    LOG(FATAL) << "curl_global_cleanup failed: " << status;
+  }
+
+  return EXIT_SUCCESS;
+}

oss-internship-2020/curl/examples/example4.cc

@@ -0,0 +1,128 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Sandboxed version of multi-poll.c
+// HTTP GET request with polling
+
+#include <cstdlib>
+
+#include "../sandbox.h"      // NOLINT(build/include)
+#include "curl_sapi.sapi.h"  // NOLINT(build/include)
+#include "sandboxed_api/util/flag.h"
+
+int main(int argc, char* argv[]) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  google::InitGoogleLogging(argv[0]);
+
+  absl::Status status;
+
+  // Initialize sandbox2 and sapi
+  CurlSapiSandbox sandbox;
+  status = sandbox.Init();
+  if (!status.ok()) {
+    LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status;
+  }
+  CurlApi api(&sandbox);
+
+  // Number of running handles
+  sapi::v::Int still_running(1);
+
+  absl::StatusOr<int> curl_code;
+
+  // Initialize curl (CURL_GLOBAL_DEFAULT = 3)
+  curl_code = api.curl_global_init(3l);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_global_init failed: " << curl_code.status();
+  }
+
+  // Initialize http_handle
+  absl::StatusOr<CURL*> curl_handle = api.curl_easy_init();
+  if (!curl_handle.ok()) {
+    LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status();
+  }
+  sapi::v::RemotePtr http_handle(curl_handle.value());
+  if (!http_handle.GetValue()) {
+    LOG(FATAL) << "curl_easy_init failed: http_handle is NULL";
+  }
+
+  // Specify URL to get
+  sapi::v::ConstCStr url("http://example.com");
+  curl_code =
+      api.curl_easy_setopt_ptr(&http_handle, CURLOPT_URL, url.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Initialize multi_handle
+  absl::StatusOr<CURLM*> curlm_handle = api.curl_multi_init();
+  if (!curlm_handle.ok()) {
+    LOG(FATAL) << "curl_multi_init failed: " << curlm_handle.status();
+  }
+  sapi::v::RemotePtr multi_handle(curlm_handle.value());
+  if (!multi_handle.GetValue()) {
+    LOG(FATAL) << "curl_multi_init failed: multi_handle is NULL";
+  }
+
+  // Add http_handle to the multi stack
+  curl_code = api.curl_multi_add_handle(&multi_handle, &http_handle);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_multi_add_handle failed: " << curl_code.status();
+  }
+
+  while (still_running.GetValue()) {
+    sapi::v::Int numfds(0);
+
+    // Perform the request
+    curl_code = api.curl_multi_perform(&multi_handle, still_running.PtrBoth());
+    if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+      LOG(FATAL) << "curl_mutli_perform failed: " << curl_code.status();
+    }
+
+    if (still_running.GetValue()) {
+      // Wait for an event or timeout
+      sapi::v::NullPtr null_ptr;
+      curl_code = api.curl_multi_poll_sapi(&multi_handle, &null_ptr, 0, 1000,
+                                           numfds.PtrBoth());
+      if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+        LOG(FATAL) << "curl_multi_poll_sapi failed: " << curl_code.status();
+      }
+    }
+  }
+
+  // Remove http_handle from the multi stack
+  curl_code = api.curl_multi_remove_handle(&multi_handle, &http_handle);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_multi_remove_handle failed: " << curl_code.status();
+  }
+
+  // Cleanup http_handle
+  status = api.curl_easy_cleanup(&http_handle);
+  if (!status.ok()) {
+    LOG(FATAL) << "curl_easy_cleanup failed: " << status;
+  }
+
+  // Cleanup multi_handle
+  curl_code = api.curl_multi_cleanup(&multi_handle);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_multi_cleanup failed: " << curl_code.status();
+  }
+
+  // Cleanup curl
+  status = api.curl_global_cleanup();
+  if (!status.ok()) {
+    LOG(FATAL) << "curl_global_cleanup failed: " << status;
+  }
+
+  return EXIT_SUCCESS;
+}

oss-internship-2020/curl/examples/example5.cc

@@ -0,0 +1,102 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Sandboxed version of multithread.c
+// Multithreaded HTTP GET requests
+
+#include <pthread.h>
+
+#include <cstdlib>
+
+#include "../sandbox.h"  // NOLINT(build/include)
+
+void pull_one_url(const std::string& url, CurlApi& api) {
+  // Initialize the curl session
+  absl::StatusOr<CURL*> curl_handle = api.curl_easy_init();
+  if (!curl_handle.ok()) {
+    LOG(FATAL) << "curl_easy_init failed: " << curl_handle.status();
+  }
+  sapi::v::RemotePtr curl(curl_handle.value());
+  if (!curl.GetValue()) {
+    LOG(FATAL) << "curl_easy_init failed: curl is NULL";
+  }
+
+  absl::StatusOr<int> curl_code;
+
+  // Specify URL to get
+  sapi::v::ConstCStr sapi_url(url.c_str());
+  curl_code =
+      api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, sapi_url.PtrBefore());
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_setopt_ptr failed: " << curl_code.status();
+  }
+
+  // Perform the request
+  curl_code = api.curl_easy_perform(&curl);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_easy_perform failed: " << curl_code.status();
+  }
+
+  // Cleanup curl
+  absl::Status status = api.curl_easy_cleanup(&curl);
+  if (!status.ok()) {
+    LOG(FATAL) << "curl_easy_cleanup failed: " << status;
+  }
+}
+
+const std::vector<std::string> urls = {
+    "http://example.com", "http://example.edu", "http://example.net",
+    "http://example.org"};
+
+int main(int argc, char* argv[]) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  google::InitGoogleLogging(argv[0]);
+
+  absl::Status status;
+
+  // Initialize sandbox2 and sapi
+  CurlSapiSandbox sandbox;
+  status = sandbox.Init();
+  if (!status.ok()) {
+    LOG(FATAL) << "Couldn't initialize Sandboxed API: " << status;
+  }
+  CurlApi api(&sandbox);
+
+  absl::StatusOr<int> curl_code;
+
+  // Initialize curl (CURL_GLOBAL_DEFAULT = 3)
+  curl_code = api.curl_global_init(3l);
+  if (!curl_code.ok() || curl_code.value() != CURLE_OK) {
+    LOG(FATAL) << "curl_global_init failed: " << curl_code.status();
+  }
+
+  // Create the threads
+  std::vector<std::thread> threads;
+  for (auto& url : urls) {
+    threads.emplace_back(pull_one_url, std::ref(url), std::ref(api));
+  }
+
+  // Join the threads
+  for (auto& thread : threads) {
+    thread.join();
+  }
+
+  // Cleanup curl
+  status = api.curl_global_cleanup();
+  if (!status.ok()) {
+    LOG(FATAL) << "curl_global_cleanup failed: " << status;
+  }
+
+  return EXIT_SUCCESS;
+}

oss-internship-2020/curl/examples/example6.cc

@@ -0,0 +1,72 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Sandboxed version of simple.c using transactions
+// Simple HTTP GET request
+
+#include <cstdlib>
+
+#include "../sandbox.h"  // NOLINT(build/include)
+#include "sandboxed_api/transaction.h"
+
+class CurlTransaction : public sapi::Transaction {
+ public:
+  explicit CurlTransaction(std::unique_ptr<sapi::Sandbox> sandbox)
+      : sapi::Transaction(std::move(sandbox)) {
+    sapi::Transaction::SetTimeLimit(kTimeOutVal);
+  }
+
+ private:
+  // Default timeout value for each transaction run.
+  static constexpr time_t kTimeOutVal = 2;
+
+  // The main processing function.
+  absl::Status Main() override;
+};
+
+absl::Status CurlTransaction::Main() {
+  CurlApi api(sandbox());
+
+  // Initialize the curl session
+  SAPI_ASSIGN_OR_RETURN(void* curl_remote, api.curl_easy_init());
+  sapi::v::RemotePtr curl(curl_remote);
+  TRANSACTION_FAIL_IF_NOT(curl.GetValue(), "curl_easy_init failed");
+
+  // Specify URL to get
+  sapi::v::ConstCStr url("http://example.com");
+  SAPI_ASSIGN_OR_RETURN(
+      int setopt_url_code,
+      api.curl_easy_setopt_ptr(&curl, CURLOPT_URL, url.PtrBefore()));
+  TRANSACTION_FAIL_IF_NOT(setopt_url_code == CURLE_OK,
+                          "curl_easy_setopt_ptr failed");
+
+  // Perform the request
+  SAPI_ASSIGN_OR_RETURN(int perform_code, api.curl_easy_perform(&curl));
+  TRANSACTION_FAIL_IF_NOT(setopt_url_code == CURLE_OK,
+                          "curl_easy_perform failed");
+
+  // Cleanup curl
+  TRANSACTION_FAIL_IF_NOT(api.curl_easy_cleanup(&curl).ok(),
+                          "curl_easy_cleanup failed");
+
+  return absl::OkStatus();
+}
+
+int main(int argc, char* argv[]) {
+  CurlTransaction curl{std::make_unique<CurlSapiSandbox>()};
+  absl::Status status = curl.Run();
+  CHECK(status.ok()) << "CurlTransaction failed";
+
+  return EXIT_SUCCESS;
+}

oss-internship-2020/curl/sandbox.h

@@ -0,0 +1,67 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SANDBOX_H_
+#define SANDBOX_H_
+
+#include <linux/futex.h>
+#include <sys/mman.h>  // For mmap arguments
+#include <syscall.h>
+
+#include <cstdlib>
+
+#include "curl_sapi.sapi.h"  // NOLINT(build/include)
+#include "sandboxed_api/sandbox2/util/bpf_helper.h"
+
+class CurlSapiSandbox : public CurlSandbox {
+ protected:
+  std::unique_ptr<sandbox2::Policy> ModifyPolicy(
+      sandbox2::PolicyBuilder* policy_builder) override {
+    // Return a new policy
+    return (*policy_builder)
+        .AllowDynamicStartup()
+        .AllowExit()
+        .AllowFork()
+        .AllowFutexOp(FUTEX_WAKE_PRIVATE)
+        .AllowMmap()
+        .AllowOpen()
+        .AllowRead()
+        .AllowSafeFcntl()
+        .AllowWrite()
+        .AllowSyscalls({
+            __NR_accept,
+            __NR_access,
+            __NR_bind,
+            __NR_connect,
+            __NR_getpeername,
+            __NR_getsockname,
+            __NR_getsockopt,
+            __NR_ioctl,
+            __NR_listen,
+            __NR_madvise,
+            __NR_poll,
+            __NR_recvfrom,
+            __NR_recvmsg,
+            __NR_sendmmsg,
+            __NR_sendto,
+            __NR_setsockopt,
+            __NR_socket,
+        })
+        .AllowUnrestrictedNetworking()
+        .AddDirectory("/lib")
+        .BuildOrDie();
+  }
+};
+
+#endif  // SANDBOX_H_

oss-internship-2020/curl/tests/CMakeLists.txt

@@ -0,0 +1,28 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include(GoogleTest)
+
+add_executable(tests
+  test_utils.h
+  test_utils.cc
+  tests.cc
+)
+
+target_link_libraries(tests
+  curl_sapi sapi::sapi
+  gtest gmock gtest_main
+)
+
+gtest_discover_tests(tests)

oss-internship-2020/curl/tests/test_utils.cc

@@ -0,0 +1,301 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_utils.h"  // NOLINT(build/include)
+
+#include <absl/strings/match.h>
+#include <fcntl.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <memory>
+#include <thread>  // NOLINT(build/c++11)
+
+int CurlTestUtils::port_;
+std::thread CurlTestUtils::server_thread_;
+
+absl::Status CurlTestUtils::CurlTestSetUp() {
+  // Initialize sandbox2 and sapi
+  sandbox_ = std::make_unique<CurlSapiSandbox>();
+  absl::Status init = sandbox_->Init();
+  if (!init.ok()) {
+    return init;
+  }
+  api_ = std::make_unique<CurlApi>(sandbox_.get());
+
+  // Initialize curl
+  absl::StatusOr<CURL*> curl_handle = api_->curl_easy_init();
+  if (!curl_handle.ok()) {
+    return curl_handle.status();
+  }
+  if (!curl_handle.value()) {
+    return absl::UnavailableError("curl_easy_init returned NULL ");
+  }
+  curl_ = std::make_unique<sapi::v::RemotePtr>(curl_handle.value());
+
+  absl::StatusOr<int> curl_code;
+
+  // Specify request URL
+  sapi::v::ConstCStr sapi_url(kUrl.data());
+  curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_URL,
+                                         sapi_url.PtrBefore());
+  if (!curl_code.ok()) {
+    return curl_code.status();
+  }
+  if (curl_code.value() != CURLE_OK) {
+    return absl::UnavailableError(
+        "curl_easy_setopt_ptr returned with the error code " +
+        curl_code.value());
+  }
+
+  // Set port
+  curl_code = api_->curl_easy_setopt_long(curl_.get(), CURLOPT_PORT, port_);
+  if (!curl_code.ok()) {
+    return curl_code.status();
+  }
+  if (curl_code.value() != CURLE_OK) {
+    return absl::UnavailableError(
+        "curl_easy_setopt_long returned with the error code " +
+        curl_code.value());
+  }
+
+  // Generate pointer to the WriteToMemory callback
+  void* function_ptr;
+  absl::Status symbol =
+      sandbox_->rpc_channel()->Symbol("WriteToMemory", &function_ptr);
+  if (!symbol.ok()) {
+    return symbol;
+  }
+  sapi::v::RemotePtr remote_function_ptr(function_ptr);
+
+  // Set WriteToMemory as the write function
+  curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_WRITEFUNCTION,
+                                         &remote_function_ptr);
+  if (!curl_code.ok()) {
+    return curl_code.status();
+  }
+  if (curl_code.value() != CURLE_OK) {
+    return absl::UnavailableError(
+        "curl_easy_setopt_ptr returned with the error code " +
+        curl_code.value());
+  }
+
+  // Pass memory chunk object to the callback
+  chunk_ = std::make_unique<sapi::v::LenVal>(0);
+  curl_code = api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_WRITEDATA,
+                                         chunk_->PtrBoth());
+  if (!curl_code.ok()) {
+    return curl_code.status();
+  }
+  if (curl_code.value() != CURLE_OK) {
+    return absl::UnavailableError(
+        "curl_easy_setopt_ptr returned with the error code " +
+        curl_code.value());
+  }
+
+  return absl::OkStatus();
+}
+
+absl::Status CurlTestUtils::CurlTestTearDown() {
+  // Cleanup curl
+  return api_->curl_easy_cleanup(curl_.get());
+}
+
+absl::StatusOr<std::string> CurlTestUtils::PerformRequest() {
+  // Perform the request
+  absl::StatusOr<int> curl_code = api_->curl_easy_perform(curl_.get());
+  if (!curl_code.ok()) {
+    return curl_code.status();
+  }
+  if (curl_code.value() != CURLE_OK) {
+    return absl::UnavailableError(
+        "curl_easy_perform returned with the error code " + curl_code.value());
+  }
+
+  // Get pointer to the memory chunk
+  absl::Status status = sandbox_->TransferFromSandboxee(chunk_.get());
+  if (!status.ok()) {
+    return status;
+  }
+
+  return std::string{reinterpret_cast<char*>(chunk_->GetData())};
+}
+
+namespace {
+
+// Read the socket until str is completely read
+std::string ReadUntil(const int socket, const std::string& str,
+                      const size_t max_request_size) {
+  std::string str_read;
+  str_read.reserve(max_request_size);
+
+  // Read one char at a time until str is suffix of buf
+  while (!absl::EndsWith(str_read, str)) {
+    char next_char;
+    if (str_read.size() >= max_request_size ||
+        read(socket, &next_char, 1) < 1) {
+      return "";
+    }
+    str_read += next_char;
+  }
+
+  return str_read;
+}
+
+// Parse HTTP headers to return the Content-Length
+ssize_t GetContentLength(const std::string& headers) {
+  // Find the Content-Length header
+  std::string::size_type length_header_start = headers.find("Content-Length: ");
+
+  // There is no Content-Length field
+  if (length_header_start == std::string::npos) {
+    return 0;
+  }
+
+  // Find Content-Length string
+  std::string::size_type length_start =
+      length_header_start + std::string{"Content-Length: "}.size();
+  std::string::size_type length_bytes =
+      headers.find("\r\n", length_start) - length_start;
+
+  // length_bytes exceeds maximum
+  if (length_bytes >= 64) {
+    return -1;
+  }
+
+  // Convert string to int and return
+  return std::stoi(headers.substr(length_start, length_bytes));
+}
+
+// Read exactly content_bytes from the socket
+std::string ReadExact(int socket, size_t content_bytes) {
+  std::string str_read;
+  str_read.reserve(content_bytes);
+
+  // Read one char at a time until all chars are read
+  while (str_read.size() < content_bytes) {
+    char next_char;
+    if (read(socket, &next_char, 1) < 1) {
+      return "";
+    }
+    str_read += next_char;
+  }
+
+  return str_read;
+}
+
+// Listen on the socket and answer back to requests
+void ServerLoop(int listening_socket, sockaddr_in socket_address) {
+  socklen_t socket_address_size = sizeof(socket_address);
+
+  // Listen on the socket (maximum 1 connection)
+  if (listen(listening_socket, 1) == -1) {
+    return;
+  }
+
+  // Keep accepting connections until the thread is terminated
+  // (i.e. server_thread_ is assigned to a new thread or destroyed)
+  for (;;) {
+    // File descriptor to the connection socket
+    // This blocks the thread until a connection is established
+    int accepted_socket =
+        accept(listening_socket, reinterpret_cast<sockaddr*>(&socket_address),
+               reinterpret_cast<socklen_t*>(&socket_address_size));
+    if (accepted_socket == -1) {
+      return;
+    }
+
+    constexpr int kMaxRequestSize = 4096;
+
+    // Read until the end of the headers
+    std::string headers =
+        ReadUntil(accepted_socket, "\r\n\r\n", kMaxRequestSize);
+
+    if (headers == "") {
+      close(accepted_socket);
+      return;
+    }
+
+    // Get the length of the request content
+    ssize_t content_length = GetContentLength(headers);
+    if (content_length > kMaxRequestSize - headers.size() ||
+        content_length < 0) {
+      close(accepted_socket);
+      return;
+    }
+
+    // Read the request content
+    std::string content = ReadExact(accepted_socket, content_length);
+
+    // Prepare a response for the request
+    std::string http_response =
+        "HTTP/1.1 200 OK\nContent-Type: text/plain\nContent-Length: ";
+
+    if (headers.substr(0, 3) == "GET") {
+      http_response += "2\r\n\r\nOK";
+
+    } else if (headers.substr(0, 4) == "POST") {
+      http_response +=
+          std::to_string(content.size()) + "\r\n\r\n" + std::string{content};
+
+    } else {
+      close(accepted_socket);
+      return;
+    }
+
+    // Ignore any errors, the connection will be closed anyway
+    write(accepted_socket, http_response.c_str(), http_response.size());
+
+    // Close the socket
+    close(accepted_socket);
+  }
+}
+
+}  // namespace
+
+void CurlTestUtils::StartMockServer() {
+  // Get the socket file descriptor
+  int listening_socket = socket(AF_INET, SOCK_STREAM, 0);
+
+  // Create the socket address object
+  // The port is set to 0, meaning that it will be auto assigned
+  // Only local connections can access this socket
+  sockaddr_in socket_address{AF_INET, 0, htonl(INADDR_LOOPBACK)};
+  socklen_t socket_address_size = sizeof(socket_address);
+  if (listening_socket == -1) {
+    return;
+  }
+
+  // Bind the file descriptor to the socket address object
+  if (bind(listening_socket, reinterpret_cast<sockaddr*>(&socket_address),
+           socket_address_size) == -1) {
+    return;
+  }
+
+  // Assign an available port to the socket address object
+  if (getsockname(listening_socket,
+                  reinterpret_cast<sockaddr*>(&socket_address),
+                  &socket_address_size) == -1) {
+    return;
+  }
+
+  // Get the port number
+  port_ = ntohs(socket_address.sin_port);
+
+  // Set server_thread_ operation to socket listening
+  server_thread_ = std::thread(ServerLoop, listening_socket, socket_address);
+}

oss-internship-2020/curl/tests/test_utils.h

@@ -0,0 +1,55 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TESTS_H_
+#define TESTS_H_
+
+#include "../sandbox.h"      // NOLINT(build/include)
+#include "curl_sapi.sapi.h"  // NOLINT(build/include)
+#include "gtest/gtest.h"
+#include "sandboxed_api/util/flag.h"
+#include "sandboxed_api/util/status_matchers.h"
+
+// Helper class that can be used to test Curl Sandboxed
+class CurlTestUtils {
+ protected:
+  // Initialize and set up the curl handle
+  absl::Status CurlTestSetUp();
+  // Clean up the curl handle
+  absl::Status CurlTestTearDown();
+
+  // Perform a request to the mock server, return the response
+  absl::StatusOr<std::string> PerformRequest();
+
+  // Start a mock server (only once) that will manage connections for the tests
+  // The server listens on a port asynchronously by creating a thread
+  // The port number is stored in port_
+  // Responds with "OK" to a GET request
+  // Responds with the POST request fields to a POST request
+  static void StartMockServer();
+
+  std::unique_ptr<CurlSapiSandbox> sandbox_;
+  std::unique_ptr<CurlApi> api_;
+  std::unique_ptr<sapi::v::RemotePtr> curl_;
+
+  static std::thread server_thread_;
+
+  static constexpr absl::string_view kUrl = "http://127.0.0.1/";
+  static int port_;
+
+ private:
+  std::unique_ptr<sapi::v::LenVal> chunk_;
+};
+
+#endif  // TESTS_H_

oss-internship-2020/curl/tests/tests.cc

@@ -0,0 +1,138 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "test_utils.h"  // NOLINT(build/include)
+
+class CurlTest : public CurlTestUtils, public ::testing::Test {
+ protected:
+  void SetUp() override {
+    // Start mock server, get port number and check for any error
+    StartMockServer();
+    ASSERT_TRUE(server_thread_.joinable());
+    ASSERT_TRUE(CurlTestSetUp().ok());
+  }
+
+  void TearDown() override {
+    ASSERT_TRUE(CurlTestTearDown().ok());
+    // Detach the server thread
+    server_thread_.detach();
+  }
+};
+
+TEST_F(CurlTest, EffectiveUrl) {
+  sapi::v::RemotePtr effective_url_ptr(nullptr);
+
+  ASSERT_TRUE(PerformRequest().ok());
+
+  // Get effective URL
+  SAPI_ASSERT_OK_AND_ASSIGN(
+      int getinfo_code,
+      api_->curl_easy_getinfo_ptr(curl_.get(), CURLINFO_EFFECTIVE_URL,
+                                  effective_url_ptr.PtrBoth()));
+  ASSERT_EQ(getinfo_code, CURLE_OK);
+
+  // Store effective URL in a string
+  SAPI_ASSERT_OK_AND_ASSIGN(std::string effective_url,
+                       sandbox_->GetCString(sapi::v::RemotePtr(
+                           effective_url_ptr.GetPointedVar())));
+
+  // Compare effective URL with original URL
+  ASSERT_EQ(effective_url, kUrl);
+}
+
+TEST_F(CurlTest, EffectivePort) {
+  sapi::v::Int effective_port;
+
+  ASSERT_TRUE(PerformRequest().ok());
+
+  // Get effective port
+  SAPI_ASSERT_OK_AND_ASSIGN(int getinfo_code, api_->curl_easy_getinfo_ptr(
+                                             curl_.get(), CURLINFO_PRIMARY_PORT,
+                                             effective_port.PtrBoth()));
+  ASSERT_EQ(getinfo_code, CURLE_OK);
+
+  // Compare effective port with port set by the mock server
+  ASSERT_EQ(effective_port.GetValue(), port_);
+}
+
+TEST_F(CurlTest, ResponseCode) {
+  sapi::v::Int response_code;
+
+  ASSERT_TRUE(PerformRequest().ok());
+
+  // Get response code
+  SAPI_ASSERT_OK_AND_ASSIGN(
+      int getinfo_code,
+      api_->curl_easy_getinfo_ptr(curl_.get(), CURLINFO_RESPONSE_CODE,
+                                  response_code.PtrBoth()));
+  ASSERT_EQ(getinfo_code, CURLE_OK);
+
+  // Check response code
+  ASSERT_EQ(response_code.GetValue(), 200);
+}
+
+TEST_F(CurlTest, ContentType) {
+  sapi::v::RemotePtr content_type_ptr(nullptr);
+
+  ASSERT_TRUE(PerformRequest().ok());
+
+  // Get effective URL
+  SAPI_ASSERT_OK_AND_ASSIGN(int getinfo_code, api_->curl_easy_getinfo_ptr(
+                                             curl_.get(), CURLINFO_CONTENT_TYPE,
+                                             content_type_ptr.PtrBoth()));
+  ASSERT_EQ(getinfo_code, CURLE_OK);
+
+  // Store content type in a string
+  SAPI_ASSERT_OK_AND_ASSIGN(std::string content_type,
+                       sandbox_->GetCString(sapi::v::RemotePtr(
+                           content_type_ptr.GetPointedVar())));
+
+  // Compare content type with "text/plain"
+  ASSERT_EQ(content_type, "text/plain");
+}
+
+TEST_F(CurlTest, GETResponse) {
+  SAPI_ASSERT_OK_AND_ASSIGN(std::string response, PerformRequest());
+
+  // Compare response with expected response
+  ASSERT_EQ(response, "OK");
+}
+
+TEST_F(CurlTest, POSTResponse) {
+  sapi::v::ConstCStr post_fields("postfields");
+
+  // Set request method to POST
+  SAPI_ASSERT_OK_AND_ASSIGN(int setopt_post, api_->curl_easy_setopt_long(
+                                            curl_.get(), CURLOPT_POST, 1l));
+  ASSERT_EQ(setopt_post, CURLE_OK);
+
+  // Set the size of the POST fields
+  SAPI_ASSERT_OK_AND_ASSIGN(
+      int setopt_post_fields_size,
+      api_->curl_easy_setopt_long(curl_.get(), CURLOPT_POSTFIELDSIZE,
+                                  post_fields.GetSize()));
+  ASSERT_EQ(setopt_post_fields_size, CURLE_OK);
+
+  // Set the POST fields
+  SAPI_ASSERT_OK_AND_ASSIGN(
+      int setopt_post_fields,
+      api_->curl_easy_setopt_ptr(curl_.get(), CURLOPT_POSTFIELDS,
+                                 post_fields.PtrBefore()));
+  ASSERT_EQ(setopt_post_fields, CURLE_OK);
+
+  SAPI_ASSERT_OK_AND_ASSIGN(std::string response, PerformRequest());
+
+  // Compare response with expected response
+  ASSERT_EQ(std::string(post_fields.GetData()), response);
+}

oss-internship-2020/gdal/CMakeLists.txt

@@ -0,0 +1,86 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.10)
+
+project(test CXX C)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+find_package(PNG REQUIRED)
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(libpcre REQUIRED IMPORTED_TARGET libpcre)
+pkg_check_modules(proj REQUIRED IMPORTED_TARGET proj)
+
+set(SAPI_ROOT "${PROJECT_SOURCE_DIR}/../.." CACHE PATH "Path to the Sandboxed API source tree")
+#   cmake .. -G Ninja -DSAPI_ROOT=$HOME/sapi_root
+
+set(SAPI_ENABLE_EXAMPLES OFF CACHE BOOL "")
+set(SAPI_ENABLE_TESTS OFF CACHE BOOL "")
+add_subdirectory("${SAPI_ROOT}"
+                 "${CMAKE_BINARY_DIR}/sandboxed-api-build"
+                 # Omit this to have the full Sandboxed API in IDE
+                 EXCLUDE_FROM_ALL)
+
+add_library(libgdal STATIC IMPORTED)
+set_property(TARGET libgdal PROPERTY IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/lib/libgdal.a")
+
+target_link_libraries(libgdal INTERFACE
+  crypto
+  expat
+  jpeg
+  PkgConfig::libpcre
+  PkgConfig::proj
+  sqlite3
+  tiff
+  z
+  pthread
+  m
+  rt
+  dl
+  curl
+  PNG::PNG
+)
+
+add_sapi_library(gdal_sapi
+  FUNCTIONS GDALOpen
+            GDALAllRegister
+            GDALGetDatasetDriver
+            GDALGetDriverShortName
+            GDALGetDriverLongName
+            GDALGetGeoTransform
+            GDALGetRasterBand
+            GDALGetBlockSize
+            GDALGetRasterBandXSize
+            GDALGetRasterBandYSize
+            GDALRasterIO
+  INPUTS "/usr/include/gdal/gdal.h"
+  LIBRARY libgdal
+  LIBRARY_NAME GDAL
+  NAMESPACE ""
+)
+
+target_include_directories(gdal_sapi INTERFACE
+  "${PROJECT_BINARY_DIR}"
+)
+
+add_executable(raster
+  raster.cc
+)
+
+target_link_libraries(raster
+  gdal_sapi
+  sapi::sapi
+)

oss-internship-2020/gdal/README.md

@@ -0,0 +1,88 @@
+# GDAL Raster GeoTIFF Workflow
+
+```
+Build Tools: CMake/Ninja
+OS: Linux
+```
+
+### For installing GDAL:
+
+```
+sudo apt-get install python3.6-dev
+sudo add-apt-repository ppa:ubuntugis/ppa && sudo apt update
+sudo apt-get install gdal-bin
+sudo apt-get install libgdal-dev
+```
+
+### Dependencies:
+
+PNG: `sudo apt-get install libpng-dev`
+
+PCRE: `sudo apt-get install libpcre3 libpcre3-dev`
+
+PROJ: `sudo apt-get install libproj-dev`
+
+OBS! You may need to set `export LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib`.
+It is required for libproj.so to be found into /usr/local/lib/. You can also fix
+this by typing `locate libproj.so` which will give you
+<the_absolute_libproj.so_path> and then `cp <the_absolute_libproj.so_path>
+/usr/local/lib/`.
+
+### Initializing GDAL submodule:
+
+`git submodule add https://github.com/OSGeo/gdal/tree/master/gdal`
+
+### Building GDAL statically
+
+GNUmakefile from gdal/gdal can handle building the static library.
+
+`cd gdal/gdal && make static-lib`
+
+`cd ../.. && mkdir lib`
+
+`cp gdal/gdal/libgdal.a lib/`
+
+OBS! The file is huge! It may take a while.
+
+### For testing:
+
+`mkdir build && cd build`
+
+`cmake .. -G Ninja`
+
+`ninja`
+
+`./raster <your_absolute_tiff_file_path>`
+
+## About the project
+
+GDAL is a translator library for raster and vector geospatial data format. The
+project consist in rastering a GeoTIFF file format using GDAL functionalities
+and sandboxed methods.
+
+## Implementation
+
+*Sandboxing...*
+
+The purpose of sandboxing is to limit the permissions and capabilities of
+library’s methods, in order to secure the usage of them. After obtaining the
+sandbox, the functions will be called through an Sandbox API (being called api
+in the current test) and so, the operations, system calls or namspaces access
+may be controlled.
+
+*Raster process...*
+
+Useful functions from the `gdal.h` header are added to the SAPI library built
+with CMake.
+
+One .tiff file is manipulated with GDALOpen functionality, which extracts a
+pointer to the data set containg a list of raster bands, all pertaining to the
+same area. Metadata, a coordinate system, a georeferencing transform, size of
+raster and various other information are kept into the data set that corresponds
+to the image.
+
+To create an array containing the image information, the dimentions needed are
+extracted using some specific GDAL(X/Y)Size functions applied to the block.
+GDALRasterBand function takes care of data type conversion, one more step
+following: placing the converted data (with RasterIO method) into the created
+and well allocated structure.

oss-internship-2020/gdal/raster.cc

@@ -0,0 +1,195 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gflags/gflags.h>
+#include <syscall.h>
+
+#include <fstream>
+#include <iostream>
+
+#include <glog/logging.h>
+#include "gdal_sapi.sapi.h"  // NOLINT(build/include)
+#include "sandboxed_api/sandbox2/util/fileops.h"
+
+class GdalSapiSandbox : public GDALSandbox {
+ public:
+  GdalSapiSandbox(std::string path)
+      : GDALSandbox(), file_path_(std::move(path)) {}
+
+  std::unique_ptr<sandbox2::Policy> ModifyPolicy(
+      sandbox2::PolicyBuilder*) override {
+    return sandbox2::PolicyBuilder()
+        .AllowDynamicStartup()
+        .AllowRead()
+        .AllowSystemMalloc()
+        .AllowWrite()
+        .AllowExit()
+        .AllowStat()
+        .AllowOpen()
+        .AllowSyscalls({
+            __NR_futex,
+            __NR_close,
+            __NR_recvmsg,
+            __NR_getdents64,
+            __NR_lseek,
+            __NR_getpid,
+            __NR_sysinfo,
+            __NR_prlimit64,
+            __NR_ftruncate,
+            __NR_unlink,
+        })
+        .AddFile(file_path_)
+        .BuildOrDie();
+  }
+
+ private:
+  std::string file_path_;
+};
+
+absl::Status GdalMain(std::string filename) {
+  // Reading GDALDataset from a (local, specific) file.
+  GdalSapiSandbox sandbox(filename);
+
+  SAPI_RETURN_IF_ERROR(sandbox.Init());
+  GDALApi api(&sandbox);
+
+  sapi::v::CStr s(filename.data());
+
+  SAPI_RETURN_IF_ERROR(api.GDALAllRegister());
+  auto open = api.GDALOpen(s.PtrBoth(), GDALAccess::GA_ReadOnly);
+
+  LOG(INFO) << "Dataset pointer adress: " << open.value() << std::endl;
+  sapi::v::RemotePtr ptr_dataset(open.value());
+
+  LOG(INFO) << ptr_dataset.ToString() << std::endl;
+  if (!open.value()) {
+    return absl::AbortedError("NULL pointer for Dataset.\n");
+  }
+
+  // Printing some general information about the dataset.
+  auto driver = api.GDALGetDatasetDriver(&ptr_dataset);
+  sapi::v::RemotePtr ptr_driver(driver.value());
+
+  auto driver_short_name = api.GDALGetDriverShortName(&ptr_driver);
+  auto driver_long_name = api.GDALGetDriverLongName(&ptr_driver);
+
+  sapi::v::RemotePtr ptr_driver_short_name(driver_short_name.value());
+  sapi::v::RemotePtr ptr_driver_long_name(driver_long_name.value());
+
+  LOG(INFO) << "Driver short name: "
+            << sandbox.GetCString(ptr_driver_short_name).value().c_str();
+  LOG(INFO) << "Driver long name: "
+            << sandbox.GetCString(ptr_driver_long_name).value().c_str();
+
+  // Checking that GetGeoTransform is valid.
+  std::vector<double> adf_geo_transform(6);
+  sapi::v::Array<double> adf_geo_transform_array(&adf_geo_transform[0],
+                                                 adf_geo_transform.size());
+
+  // For this function that returns CPLErr, the error-handling must be done
+  // analyzing the returning object.
+  // Same for GDALReturnsIO from below.
+  CPLErr err;
+  SAPI_ASSIGN_OR_RETURN(err, api.GDALGetGeoTransform(
+                            &ptr_dataset, adf_geo_transform_array.PtrBoth()));
+
+  // If GDALGetGeoTransform generates an error.
+  if (err != CE_None) {
+    return absl::InternalError("GDAL rasterization failed.");
+  }
+
+  LOG(INFO) << "Origin = (" << adf_geo_transform[0] << ", "
+            << adf_geo_transform[3] << ")" << std::endl;
+  LOG(INFO) << "Pixel Size = (" << adf_geo_transform[0] << ", "
+            << adf_geo_transform[3] << ")" << std::endl;
+
+  std::vector<int> n_blockX_size(1);
+  std::vector<int> n_blockY_size(1);
+
+  sapi::v::Array<int> nBlockXSizeArray(&n_blockX_size[0], n_blockX_size.size());
+  sapi::v::Array<int> nBlockYSizeArray(&n_blockY_size[0], n_blockY_size.size());
+
+  auto band = api.GDALGetRasterBand(&ptr_dataset, 1);
+  LOG(INFO) << "Band pointer adress: " << band.value() << std::endl;
+  if (!band.value()) {
+    return absl::AbortedError("NULL pointer for Band.\n");
+  }
+
+  sapi::v::RemotePtr ptr_band(band.value());
+  SAPI_RETURN_IF_ERROR(api.GDALGetBlockSize(&ptr_band, nBlockXSizeArray.PtrBoth(),
+                                       nBlockYSizeArray.PtrBoth()));
+
+  LOG(INFO) << "Block = " << n_blockX_size[0] << " x " << n_blockY_size[0]
+            << std::endl;
+
+  std::vector<int> b_got_min(1);
+  std::vector<int> b_got_max(1);
+
+  sapi::v::Array<int> b_got_min_array(&b_got_min[0], b_got_min.size());
+  sapi::v::Array<int> b_got_max_array(&b_got_max[0], b_got_max.size());
+
+  auto nX_size = api.GDALGetRasterBandXSize(&ptr_band);
+  auto nY_size = api.GDALGetRasterBandYSize(&ptr_band);
+
+  std::vector<int8_t> raster_data(nX_size.value() * nY_size.value(), -1);
+  sapi::v::Array<int8_t> raster_data_array(&raster_data[0], raster_data.size());
+
+  // We will use CPLErr type of returning value, as before with
+  // GDALGetGeoTransorm.
+  SAPI_ASSIGN_OR_RETURN(
+      err, api.GDALRasterIO(&ptr_band, GF_Read, 0, 0, nX_size.value(),
+                            nY_size.value(), raster_data_array.PtrBoth(),
+                            nX_size.value(), nY_size.value(), GDT_Byte, 0, 0));
+
+  // If GDALRasterIO generates an error.
+  if (err != CE_None) {
+    return absl::InternalError("GDAL rasterization failed.");
+  }
+
+  std::cout << "Raster data info: " << raster_data_array.ToString()
+            << std::endl;
+
+  // To print the data content: `std::cout << raster_data_array.GetData() <<
+  // std::endl;`
+
+  return absl::OkStatus();
+}
+
+int main(int argc, char** argv) {
+  // The file to be converted should be specified in the first argument while
+  // running the program.
+  if (argc < 2) {
+    std::cout << "You need to provide a file name: ./raster "
+                 "your_tiff_file_absolute_path\n"
+                 "Example: ./raster /usr/home/username/file.tiff"
+              << std::endl;
+    return EXIT_FAILURE;
+  }
+
+  std::ifstream aux_file;
+  aux_file.open(argv[1]);
+  if (!aux_file.is_open()) {
+    std::cout << "Your file name is not valid.\nUnable to open the file."
+              << std::endl;
+    return EXIT_FAILURE;
+  }
+  std::string filename(argv[1]);
+
+  if (absl::Status status = GdalMain(filename); !status.ok()) {
+    LOG(ERROR) << "Initialization failed: " << status.ToString();
+    return EXIT_FAILURE;
+  }
+
+  return EXIT_SUCCESS;
+}

oss-internship-2020/guetzli/BUILD.bazel

@@ -24,6 +24,7 @@ cc_library(
     srcs = ["guetzli_entry_points.cc"],
     hdrs = ["guetzli_entry_points.h"],
     deps = [
+        "@com_google_absl//absl/status:statusor",
         "@com_google_sandboxed_api//sandboxed_api:lenval_core",
         "@com_google_sandboxed_api//sandboxed_api:vars",
         "@guetzli//:guetzli_lib",
@@ -53,9 +54,7 @@ sapi_library(
 cc_binary(
     name = "guetzli_sandboxed",
     srcs = ["guetzli_sandboxed.cc"],
-    deps = [
+    deps = [":guetzli_sapi"],
-        ":guetzli_sapi",
-    ],
 )
 
 cc_test(

oss-internship-2020/guetzli/guetzli_entry_points.cc

@@ -25,8 +25,8 @@
 #include "guetzli/jpeg_data_reader.h"
 #include "guetzli/quality.h"
 #include "png.h"  // NOLINT(build/include)
+#include "absl/status/statusor.h"
 #include "sandboxed_api/sandbox2/util/fileops.h"
-#include "sandboxed_api/util/statusor.h"
 
 namespace {
 
@@ -51,7 +51,7 @@ sapi::LenValStruct CreateLenValFromData(const void* data, size_t size) {
   return {size, new_data};
 }
 
-sapi::StatusOr<std::string> ReadFromFd(int fd) {
+absl::StatusOr<std::string> ReadFromFd(int fd) {
   struct stat file_data;
   int status = fstat(fd, &file_data);
 
@@ -70,9 +70,9 @@ sapi::StatusOr<std::string> ReadFromFd(int fd) {
   return result;
 }
 
-sapi::StatusOr<GuetzliInitData> PrepareDataForProcessing(
+absl::StatusOr<GuetzliInitData> PrepareDataForProcessing(
     const ProcessingParams& processing_params) {
-  sapi::StatusOr<std::string> input = ReadFromFd(processing_params.remote_fd);
+  absl::StatusOr<std::string> input = ReadFromFd(processing_params.remote_fd);
 
   if (!input.ok()) {
     return input.status();
@@ -96,7 +96,7 @@ inline uint8_t BlendOnBlack(const uint8_t val, const uint8_t alpha) {
 }
 
 // Modified version of ReadPNG from original guetzli.cc
-sapi::StatusOr<ImageData> ReadPNG(const std::string& data) {
+absl::StatusOr<ImageData> ReadPNG(const std::string& data) {
   std::vector<uint8_t> rgb;
   int xsize, ysize;
   png_structp png_ptr =

oss-internship-2020/guetzli/guetzli_sandboxed.cc

@@ -17,7 +17,6 @@
 
 #include "guetzli_transaction.h"  // NOLINT(build/include)
 #include "sandboxed_api/sandbox2/util/fileops.h"
-#include "sandboxed_api/util/statusor.h"
 
 namespace {
 

oss-internship-2020/guetzli/guetzli_sapi_test.cc

@@ -83,7 +83,7 @@ TEST_F(GuetzliSapiTest, ProcessRGB) {
   *processing_params.mutable_data() = {
       in_fd.GetRemoteFd(), 0, kDefaultQualityTarget, kDefaultMemlimitMb};
   sapi::v::LenVal output(0);
-  sapi::StatusOr<bool> processing_result =
+  absl::StatusOr<bool> processing_result =
       api_->ProcessRgb(processing_params.PtrBefore(), output.PtrBoth());
   ASSERT_TRUE(processing_result.value_or(false)) << "Error processing rgb data";
   std::string reference_data =
@@ -108,7 +108,7 @@ TEST_F(GuetzliSapiTest, ProcessJpeg) {
   *processing_params.mutable_data() = {
       in_fd.GetRemoteFd(), 0, kDefaultQualityTarget, kDefaultMemlimitMb};
   sapi::v::LenVal output(0);
-  sapi::StatusOr<bool> processing_result =
+  absl::StatusOr<bool> processing_result =
       api_->ProcessJpeg(processing_params.PtrBefore(), output.PtrBoth());
   ASSERT_TRUE(processing_result.value_or(false)) << "Error processing jpg data";
   std::string reference_data =

oss-internship-2020/guetzli/guetzli_transaction.cc

@@ -99,7 +99,7 @@ absl::Status GuetzliTransaction::LinkOutFile(int out_fd) const {
   return absl::OkStatus();
 }
 
-sapi::StatusOr<ImageType> GuetzliTransaction::GetImageTypeFromFd(int fd) const {
+absl::StatusOr<ImageType> GuetzliTransaction::GetImageTypeFromFd(int fd) const {
   static const unsigned char kPNGMagicBytes[] = {
       0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n',
   };

oss-internship-2020/guetzli/guetzli_transaction.h

@@ -48,7 +48,7 @@ class GuetzliTransaction : public sapi::Transaction {
   absl::Status Main() final;
 
   absl::Status LinkOutFile(int out_fd) const;
-  sapi::StatusOr<ImageType> GetImageTypeFromFd(int fd) const;
+  absl::StatusOr<ImageType> GetImageTypeFromFd(int fd) const;
 
   const TransactionParams params_;
   ImageType image_type_ = ImageType::kJpeg;

oss-internship-2020/openjpeg/examples/decompress_example.cc

@@ -73,12 +73,12 @@ int main(int argc, char* argv[]) {
   sapi::v::ConstCStr in_file_v(in_file.c_str());
 
   // Initialize library's main data-holders.
-  sapi::StatusOr<opj_stream_t*> stream =
+  absl::StatusOr<opj_stream_t*> stream =
       api.opj_stream_create_default_file_stream(in_file_v.PtrBefore(), 1);
   CHECK(stream.ok()) << "Stream initialization failed: " << stream.status();
   sapi::v::RemotePtr stream_pointer(stream.value());
 
-  sapi::StatusOr<opj_codec_t*> codec = api.opj_create_decompress(OPJ_CODEC_JP2);
+  absl::StatusOr<opj_codec_t*> codec = api.opj_create_decompress(OPJ_CODEC_JP2);
   CHECK(codec.ok()) << "Codec initialization failed: " << stream.status();
   sapi::v::RemotePtr codec_pointer(codec.value());
 
@@ -86,7 +86,7 @@ int main(int argc, char* argv[]) {
   status = api.opj_set_default_decoder_parameters(parameters.PtrBoth());
   CHECK(status.ok()) << "Parameters initialization failed " << status;
 
-  sapi::StatusOr<OPJ_BOOL> bool_status =
+  absl::StatusOr<OPJ_BOOL> bool_status =
       api.opj_setup_decoder(&codec_pointer, parameters.PtrBefore());
   CHECK(bool_status.ok() && bool_status.value()) << "Decoder setup failed";
 

oss-internship-2020/pffft/master

@@ -0,0 +1 @@
+Subproject commit 74d7261be17cf659d5930d4830609406bd7553e3

sandboxed_api/call.h

@@ -23,8 +23,8 @@ namespace sapi {
 namespace comms {
 
 struct ReallocRequest {
-  uint64_t old_addr;
+  uintptr_t old_addr;
-  uint64_t size;
+  size_t size;
 };
 
 // Types of TAGs used with Comms channel.

sandboxed_api/client.cc

@@ -229,7 +229,7 @@ void HandleCallMsg(const FuncCall& call, FuncRet* ret) {
 }
 
 // Handles requests to allocate memory inside the sandboxee.
-void HandleAllocMsg(const uintptr_t size, FuncRet* ret) {
+void HandleAllocMsg(const size_t size, FuncRet* ret) {
   VLOG(1) << "HandleAllocMsg: size=" << size;
 
   ret->ret_type = v::Type::kPointer;
@@ -237,16 +237,15 @@ void HandleAllocMsg(const uintptr_t size, FuncRet* ret) {
   // Memory is copied to the pointer using an API that the memory sanitizer
   // is blind to (process_vm_writev). Initialize the memory here so that
   // the sandboxed code can still be tested with the memory sanitizer.
-  ret->int_val =
+  ret->int_val = reinterpret_cast<uintptr_t>(calloc(1, size));
-      reinterpret_cast<uintptr_t>(calloc(1, static_cast<size_t>(size)));
 #else
-  ret->int_val = reinterpret_cast<uintptr_t>(malloc(static_cast<size_t>(size)));
+  ret->int_val = reinterpret_cast<uintptr_t>(malloc(size));
 #endif
   ret->success = true;
 }
 
 // Like HandleAllocMsg(), but handles requests to reallocate memory.
-void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) {
+void HandleReallocMsg(uintptr_t ptr, size_t size, FuncRet* ret) {
   VLOG(1) << "HandleReallocMsg(" << absl::StrCat(absl::Hex(ptr)) << ", " << size
           << ")";
 #ifdef MEMORY_SANITIZER
@@ -254,8 +253,8 @@ void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) {
       __sanitizer_get_allocated_size(reinterpret_cast<const void*>(ptr));
 #endif
   ret->ret_type = v::Type::kPointer;
-  ret->int_val = reinterpret_cast<uintptr_t>(
+  ret->int_val =
-      realloc(const_cast<void*>(reinterpret_cast<const void*>(ptr)), size));
+      reinterpret_cast<uintptr_t>(realloc(reinterpret_cast<void*>(ptr), size));
   ret->success = true;
 #ifdef MEMORY_SANITIZER
   // Memory is copied to the pointer using an API that the memory sanitizer
@@ -273,7 +272,7 @@ void HandleReallocMsg(uintptr_t ptr, uintptr_t size, FuncRet* ret) {
 void HandleFreeMsg(uintptr_t ptr, FuncRet* ret) {
   VLOG(1) << "HandleFreeMsg: free(0x" << absl::StrCat(absl::Hex(ptr)) << ")";
 
-  free(const_cast<void*>(reinterpret_cast<const void*>(ptr)));
+  free(reinterpret_cast<void*>(ptr));
   ret->ret_type = v::Type::kVoid;
   ret->success = true;
   ret->int_val = 0ULL;
@@ -363,14 +362,13 @@ void ServeRequest(sandbox2::Comms* comms) {
       break;
     case comms::kMsgAllocate:
       VLOG(1) << "Client::kMsgAllocate";
-      HandleAllocMsg(BytesAs<uintptr_t>(bytes), &ret);
+      HandleAllocMsg(BytesAs<size_t>(bytes), &ret);
       break;
     case comms::kMsgReallocate:
       VLOG(1) << "Client::kMsgReallocate";
       {
         auto req = BytesAs<comms::ReallocRequest>(bytes);
-        HandleReallocMsg(static_cast<uintptr_t>(req.old_addr),
+        HandleReallocMsg(req.old_addr, req.size, &ret);
-                         static_cast<uintptr_t>(req.size), &ret);
       }
       break;
     case comms::kMsgFree:

sandboxed_api/examples/hello_sapi/BUILD.bazel

@@ -40,7 +40,6 @@ sapi_library(
 cc_binary(
     name = "hello",
     srcs = ["hello_main.cc"],
-    includes = ["."],  # To find the generated header
     deps = [":hello_sapi"],
 )
 
@@ -50,7 +49,6 @@ cc_binary(
 cc_binary(
     name = "hello_transacted",
     srcs = ["hello_transacted.cc"],
-    includes = ["."],  # To find the generated header
     deps = [
         ":hello_sapi",
         "@com_google_absl//absl/memory",

sandboxed_api/examples/stringop/main_stringop.cc

@@ -134,7 +134,7 @@ TEST(StringopTest, RawStringLength) {
   ASSERT_THAT(sandbox.Init(), IsOk());
   StringopApi api(&sandbox);
   SAPI_ASSERT_OK_AND_ASSIGN(void* target_mem_ptr, api.get_raw_c_string());
-  SAPI_ASSERT_OK_AND_ASSIGN(uint64_t len,
+  SAPI_ASSERT_OK_AND_ASSIGN(size_t len,
                        sandbox.rpc_channel()->Strlen(target_mem_ptr));
   EXPECT_THAT(len, Eq(10));
 }
@@ -144,7 +144,7 @@ TEST(StringopTest, RawStringReading) {
   ASSERT_THAT(sandbox.Init(), IsOk());
   StringopApi api(&sandbox);
   SAPI_ASSERT_OK_AND_ASSIGN(void* target_mem_ptr, api.get_raw_c_string());
-  SAPI_ASSERT_OK_AND_ASSIGN(uint64_t len,
+  SAPI_ASSERT_OK_AND_ASSIGN(size_t len,
                        sandbox.rpc_channel()->Strlen(target_mem_ptr));
   EXPECT_THAT(len, Eq(10));
 

sandboxed_api/rpcchannel.cc

@@ -27,8 +27,7 @@ namespace sapi {
 absl::Status RPCChannel::Call(const FuncCall& call, uint32_t tag, FuncRet* ret,
                               v::Type exp_type) {
   absl::MutexLock lock(&mutex_);
-  if (!comms_->SendTLV(tag, sizeof(call),
+  if (!comms_->SendTLV(tag, sizeof(call), &call)) {
-                       reinterpret_cast<const uint8_t*>(&call))) {
     return absl::UnavailableError("Sending TLV value failed");
   }
   SAPI_ASSIGN_OR_RETURN(auto fret, Return(exp_type));
@@ -38,7 +37,7 @@ absl::Status RPCChannel::Call(const FuncCall& call, uint32_t tag, FuncRet* ret,
 
 absl::StatusOr<FuncRet> RPCChannel::Return(v::Type exp_type) {
   uint32_t tag;
-  uint64_t len;
+  size_t len;
   FuncRet ret;
   if (!comms_->RecvTLV(&tag, &len, &ret, sizeof(ret))) {
     return absl::UnavailableError("Receiving TLV value failed");
@@ -67,9 +66,7 @@ absl::StatusOr<FuncRet> RPCChannel::Return(v::Type exp_type) {
 
 absl::Status RPCChannel::Allocate(size_t size, void** addr) {
   absl::MutexLock lock(&mutex_);
-  uint64_t sz = size;
+  if (!comms_->SendTLV(comms::kMsgAllocate, sizeof(size), &size)) {
-  if (!comms_->SendTLV(comms::kMsgAllocate, sizeof(sz),
-                       reinterpret_cast<uint8_t*>(&sz))) {
     return absl::UnavailableError("Sending TLV value failed");
   }
 
@@ -81,12 +78,13 @@ absl::Status RPCChannel::Allocate(size_t size, void** addr) {
 absl::Status RPCChannel::Reallocate(void* old_addr, size_t size,
                                     void** new_addr) {
   absl::MutexLock lock(&mutex_);
-  comms::ReallocRequest req;
+  comms::ReallocRequest req = {
-  req.old_addr = reinterpret_cast<uint64_t>(old_addr);
+      .old_addr = reinterpret_cast<uintptr_t>(old_addr),
-  req.size = size;
+      .size = size,
+  };
 
   if (!comms_->SendTLV(comms::kMsgReallocate, sizeof(comms::ReallocRequest),
-                       reinterpret_cast<uint8_t*>(&req))) {
+                       &req)) {
     return absl::UnavailableError("Sending TLV value failed");
   }
 
@@ -103,9 +101,8 @@ absl::Status RPCChannel::Reallocate(void* old_addr, size_t size,
 
 absl::Status RPCChannel::Free(void* addr) {
   absl::MutexLock lock(&mutex_);
-  uint64_t remote = reinterpret_cast<uint64_t>(addr);
+  uintptr_t remote = reinterpret_cast<uintptr_t>(addr);
-  if (!comms_->SendTLV(comms::kMsgFree, sizeof(remote),
+  if (!comms_->SendTLV(comms::kMsgFree, sizeof(remote), &remote)) {
-                       reinterpret_cast<uint8_t*>(&remote))) {
     return absl::UnavailableError("Sending TLV value failed");
   }
 
@@ -118,8 +115,7 @@ absl::Status RPCChannel::Free(void* addr) {
 
 absl::Status RPCChannel::Symbol(const char* symname, void** addr) {
   absl::MutexLock lock(&mutex_);
-  if (!comms_->SendTLV(comms::kMsgSymbol, strlen(symname) + 1,
+  if (!comms_->SendTLV(comms::kMsgSymbol, strlen(symname) + 1, symname)) {
-                       reinterpret_cast<const uint8_t*>(symname))) {
     return absl::UnavailableError("Sending TLV value failed");
   }
 
@@ -137,9 +133,8 @@ absl::Status RPCChannel::Exit() {
 
   // Try the RPC exit sequence. But, the only thing that matters as a success
   // indicator is whether the Comms channel had been closed
-  bool unused = true;
+  comms_->SendTLV(comms::kMsgExit, 0, nullptr);
-  comms_->SendTLV(comms::kMsgExit, sizeof(unused),
+  bool unused;
-                  reinterpret_cast<uint8_t*>(&unused));
   comms_->RecvBool(&unused);
 
   if (!comms_->IsTerminated()) {
@@ -154,9 +149,7 @@ absl::Status RPCChannel::Exit() {
 
 absl::Status RPCChannel::SendFD(int local_fd, int* remote_fd) {
   absl::MutexLock lock(&mutex_);
-  bool unused = true;
+  if (!comms_->SendTLV(comms::kMsgSendFd, 0, nullptr)) {
-  if (!comms_->SendTLV(comms::kMsgSendFd, sizeof(unused),
-                       reinterpret_cast<uint8_t*>(&unused))) {
     return absl::UnavailableError("Sending TLV value failed");
   }
   if (!comms_->SendFD(local_fd)) {
@@ -173,8 +166,7 @@ absl::Status RPCChannel::SendFD(int local_fd, int* remote_fd) {
 
 absl::Status RPCChannel::RecvFD(int remote_fd, int* local_fd) {
   absl::MutexLock lock(&mutex_);
-  if (!comms_->SendTLV(comms::kMsgRecvFd, sizeof(remote_fd),
+  if (!comms_->SendTLV(comms::kMsgRecvFd, sizeof(remote_fd), &remote_fd)) {
-                       reinterpret_cast<uint8_t*>(&remote_fd))) {
     return absl::UnavailableError("Sending TLV value failed");
   }
 
@@ -191,8 +183,7 @@ absl::Status RPCChannel::RecvFD(int remote_fd, int* local_fd) {
 
 absl::Status RPCChannel::Close(int remote_fd) {
   absl::MutexLock lock(&mutex_);
-  if (!comms_->SendTLV(comms::kMsgClose, sizeof(remote_fd),
+  if (!comms_->SendTLV(comms::kMsgClose, sizeof(remote_fd), &remote_fd)) {
-                       reinterpret_cast<uint8_t*>(&remote_fd))) {
     return absl::UnavailableError("Sending TLV value failed");
   }
 
@@ -203,10 +194,9 @@ absl::Status RPCChannel::Close(int remote_fd) {
   return absl::OkStatus();
 }
 
-absl::StatusOr<uint64_t> RPCChannel::Strlen(void* str) {
+absl::StatusOr<size_t> RPCChannel::Strlen(void* str) {
   absl::MutexLock lock(&mutex_);
-  if (!comms_->SendTLV(comms::kMsgStrlen, sizeof(str),
+  if (!comms_->SendTLV(comms::kMsgStrlen, sizeof(str), &str)) {
-                       reinterpret_cast<uint8_t*>(&str))) {
     return absl::UnavailableError("Sending TLV value failed");
   }
 

sandboxed_api/rpcchannel.h

@@ -61,7 +61,7 @@ class RPCChannel {
   absl::Status Close(int remote_fd);
 
   // Returns length of a null-terminated c-style string (invokes strlen).
-  absl::StatusOr<uint64_t> Strlen(void* str);
+  absl::StatusOr<size_t> Strlen(void* str);
 
   sandbox2::Comms* comms() const { return comms_; }
 

sandboxed_api/sandbox.cc

@@ -394,7 +394,7 @@ absl::Status Sandbox::TransferFromSandboxee(v::Var* var) {
 }
 
 absl::StatusOr<std::string> Sandbox::GetCString(const v::RemotePtr& str,
-                                                uint64_t max_length) {
+                                                size_t max_length) {
   if (!is_active()) {
     return absl::UnavailableError("Sandbox not active");
   }

sandboxed_api/sandbox.h

@@ -103,8 +103,8 @@ class Sandbox {
   absl::Status TransferFromSandboxee(v::Var* var);
 
   absl::StatusOr<std::string> GetCString(const v::RemotePtr& str,
-                                         uint64_t max_length = 10ULL
+                                         size_t max_length = 10ULL
-                                                               << 20 /* 10 MiB*/
+                                                             << 20 /* 10 MiB*/
   );
 
   // Waits until the sandbox terminated and returns the result.

sandboxed_api/sandbox2/comms.cc

@@ -228,7 +228,7 @@ void Comms::Terminate() {
   }
 }
 
-bool Comms::SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes) {
+bool Comms::SendTLV(uint32_t tag, size_t length, const void* value) {
   if (length > GetMaxMsgSize()) {
     SAPI_RAW_LOG(ERROR, "Maximum TLV message size exceeded: (%u > %u)", length,
                  GetMaxMsgSize());
@@ -249,14 +249,14 @@ bool Comms::SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes) {
                 length);
   {
     absl::MutexLock lock(&tlv_send_transmission_mutex_);
-    if (!Send(reinterpret_cast<uint8_t*>(&tag), sizeof(tag))) {
+    if (!Send(&tag, sizeof(tag))) {
       return false;
     }
-    if (!Send(reinterpret_cast<uint8_t*>(&length), sizeof(length))) {
+    if (!Send(&length, sizeof(length))) {
       return false;
     }
     if (length > 0) {
-      if (!Send(bytes, length)) {
+      if (!Send(value, length)) {
         return false;
       }
     }
@@ -279,8 +279,7 @@ bool Comms::RecvString(std::string* v) {
 }
 
 bool Comms::SendString(const std::string& v) {
-  return SendTLV(kTagString, v.length(),
+  return SendTLV(kTagString, v.length(), v.c_str());
-                 reinterpret_cast<const uint8_t*>(v.c_str()));
 }
 
 bool Comms::RecvBytes(std::vector<uint8_t>* buffer) {
@@ -297,7 +296,7 @@ bool Comms::RecvBytes(std::vector<uint8_t>* buffer) {
   return true;
 }
 
-bool Comms::SendBytes(const uint8_t* v, uint64_t len) {
+bool Comms::SendBytes(const uint8_t* v, size_t len) {
   return SendTLV(kTagBytes, len, v);
 }
 
@@ -418,7 +417,7 @@ bool Comms::SendFD(int fd) {
   int* fds = reinterpret_cast<int*>(CMSG_DATA(cmsg));
   fds[0] = fd;
 
-  InternalTLV tlv = {kTagFd, sizeof(tlv.val), 0};
+  InternalTLV tlv = {kTagFd, 0};
 
   iovec iov;
   iov.iov_base = &tlv;
@@ -513,8 +512,9 @@ socklen_t Comms::CreateSockaddrUn(sockaddr_un* sun) {
   return slen;
 }
 
-bool Comms::Send(const uint8_t* bytes, uint64_t len) {
+bool Comms::Send(const void* data, size_t len) {
-  uint64_t total_sent = 0;
+  size_t total_sent = 0;
+  const char* bytes = reinterpret_cast<const char*>(data);
   const auto op = [bytes, len, &total_sent](int fd) -> ssize_t {
     PotentiallyBlockingRegion region;
     return TEMP_FAILURE_RETRY(write(fd, &bytes[total_sent], len - total_sent));
@@ -545,8 +545,9 @@ bool Comms::Send(const uint8_t* bytes, uint64_t len) {
   return true;
 }
 
-bool Comms::Recv(uint8_t* bytes, uint64_t len) {
+bool Comms::Recv(void* data, size_t len) {
-  uint64_t total_recv = 0;
+  size_t total_recv = 0;
+  char* bytes = reinterpret_cast<char*>(data);
   const auto op = [bytes, len, &total_recv](int fd) -> ssize_t {
     PotentiallyBlockingRegion region;
     return TEMP_FAILURE_RETRY(read(fd, &bytes[total_recv], len - total_recv));
@@ -573,7 +574,7 @@ bool Comms::Recv(uint8_t* bytes, uint64_t len) {
 }
 
 // Internal helper method (low level).
-bool Comms::RecvTL(uint32_t* tag, uint64_t* length) {
+bool Comms::RecvTL(uint32_t* tag, size_t* length) {
   if (!Recv(reinterpret_cast<uint8_t*>(tag), sizeof(*tag))) {
     return false;
   }
@@ -610,7 +611,7 @@ bool Comms::RecvTLV(uint32_t* tag, std::string* value) {
 template <typename T>
 bool Comms::RecvTLVGeneric(uint32_t* tag, T* value) {
   absl::MutexLock lock(&tlv_recv_transmission_mutex_);
-  uint64_t length;
+  size_t length;
   if (!RecvTL(tag, &length)) {
     return false;
   }
@@ -619,28 +620,29 @@ bool Comms::RecvTLVGeneric(uint32_t* tag, T* value) {
   return length == 0 || Recv(reinterpret_cast<uint8_t*>(value->data()), length);
 }
 
-bool Comms::RecvTLV(uint32_t* tag, uint64_t* length, void* buffer,
+bool Comms::RecvTLV(uint32_t* tag, size_t* length, void* buffer,
-                    uint64_t buffer_size) {
+                    size_t buffer_size) {
   absl::MutexLock lock(&tlv_recv_transmission_mutex_);
   if (!RecvTL(tag, length)) {
     return false;
   }
 
+  if (*length == 0) {
+    return true;
+  }
+
   if (*length > buffer_size) {
     SAPI_RAW_LOG(ERROR, "Buffer size too small (0x%x > 0x%x)", *length,
                  buffer_size);
     return false;
-  } else if (*length > 0) {
-    if (!Recv(reinterpret_cast<uint8_t*>(buffer), *length)) {
-      return false;
-    }
   }
-  return true;
+
+  return Recv(reinterpret_cast<uint8_t*>(buffer), *length);
 }
 
-bool Comms::RecvInt(void* buffer, uint64_t len, uint32_t tag) {
+bool Comms::RecvInt(void* buffer, size_t len, uint32_t tag) {
   uint32_t received_tag;
-  uint64_t received_length;
+  size_t received_length;
   if (!RecvTLV(&received_tag, &received_length, buffer, len)) {
     return false;
   }

sandboxed_api/sandbox2/comms.h

@@ -60,7 +60,7 @@ class Comms {
   static constexpr uint32_t kTagFd = 0X80000201;
 
   // Any payload size above this limit will LOG(WARNING).
-  static constexpr uint64_t kWarnMsgSize = (256ULL << 20);
+  static constexpr size_t kWarnMsgSize = (256ULL << 20);
 
   // Sandbox2-specific convention where FD=1023 is always passed to the
   // sandboxed process as a communication channel (encapsulated in the
@@ -103,9 +103,9 @@ class Comms {
   // Note: The actual size is "unlimited", although the Buffer API is more
   // efficient for large transfers. There is an arbitrary limit to ~2GiB to
   // avoid protobuf serialization issues.
-  uint64_t GetMaxMsgSize() const { return std::numeric_limits<int32_t>::max(); }
+  size_t GetMaxMsgSize() const { return std::numeric_limits<int32_t>::max(); }
 
-  bool SendTLV(uint32_t tag, uint64_t length, const uint8_t* bytes);
+  bool SendTLV(uint32_t tag, size_t length, const void* value);
   // Receive a TLV structure, the memory for the value will be allocated
   // by std::vector.
   bool RecvTLV(uint32_t* tag, std::vector<uint8_t>* value);
@@ -113,7 +113,7 @@ class Comms {
   // by std::string.
   bool RecvTLV(uint32_t* tag, std::string* value);
   // Receives a TLV value into a specified buffer without allocating memory.
-  bool RecvTLV(uint32_t* tag, uint64_t* length, void* buffer, uint64_t buffer_size);
+  bool RecvTLV(uint32_t* tag, size_t* length, void* buffer, size_t buffer_size);
 
   // Sends/receives various types of data.
   bool RecvUint8(uint8_t* v) { return RecvIntGeneric(v, kTagUint8); }
@@ -138,7 +138,7 @@ class Comms {
   bool SendString(const std::string& v);
 
   bool RecvBytes(std::vector<uint8_t>* buffer);
-  bool SendBytes(const uint8_t* v, uint64_t len);
+  bool SendBytes(const uint8_t* v, size_t len);
   bool SendBytes(const std::vector<uint8_t>& buffer);
 
   // Receives remote process credentials.
@@ -183,19 +183,18 @@ class Comms {
   struct ABSL_ATTRIBUTE_PACKED InternalTLV {
     uint32_t tag;
     uint32_t len;
-    uint64_t val;
   };
 
   // Fills sockaddr_un struct with proper values.
   socklen_t CreateSockaddrUn(sockaddr_un* sun);
 
   // Support for EINTR and size completion.
-  bool Send(const uint8_t* bytes, uint64_t len);
+  bool Send(const void* data, size_t len);
-  bool Recv(uint8_t* bytes, uint64_t len);
+  bool Recv(void* data, size_t len);
 
   // Receives tag and length. Assumes that the `tlv_transmission_mutex_` mutex
   // is locked.
-  bool RecvTL(uint32_t* tag, uint64_t* length)
+  bool RecvTL(uint32_t* tag, size_t* length)
       ABSL_EXCLUSIVE_LOCKS_REQUIRED(tlv_recv_transmission_mutex_);
 
   // T has to be a ContiguousContainer
@@ -203,7 +202,7 @@ class Comms {
   bool RecvTLVGeneric(uint32_t* tag, T* value);
 
   // Receives arbitrary integers.
-  bool RecvInt(void* buffer, uint64_t len, uint32_t tag);
+  bool RecvInt(void* buffer, size_t len, uint32_t tag);
 
   template <typename T>
   bool RecvIntGeneric(T* output, uint32_t tag) {
@@ -212,7 +211,7 @@ class Comms {
 
   template <typename T>
   bool SendGeneric(T value, uint32_t tag) {
-    return SendTLV(tag, sizeof(T), reinterpret_cast<const uint8_t*>(&value));
+    return SendTLV(tag, sizeof(T), &value);
   }
 };
 

sandboxed_api/tools/clang_generator/CMakeLists.txt

@@ -45,6 +45,7 @@ target_link_libraries(sapi_generator PUBLIC
   absl::memory
   absl::random_random
   absl::status
+  absl::statusor
   absl::strings
   clangFormat
   clangFrontendTool
@@ -59,12 +60,15 @@ add_executable(sapi_generator_tool
 )
 target_link_libraries(sapi_generator_tool PRIVATE
   sapi::base
+  sandbox2::file_helpers
   sandbox2::fileops
   sapi::generator
 )
 
 if(SAPI_ENABLE_TESTS)
   add_executable(sapi_generator_test
+    frontend_action_test_util.cc
+    frontend_action_test_util.h
     emitter_test.cc
   )
   target_link_libraries(sapi_generator_test PRIVATE

sandboxed_api/tools/clang_generator/emitter.cc

@@ -23,9 +23,13 @@
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_join.h"
 #include "absl/strings/str_replace.h"
+#include "absl/strings/string_view.h"
 #include "absl/strings/strip.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/Type.h"
+#include "clang/Format/Format.h"
 #include "sandboxed_api/tools/clang_generator/diagnostics.h"
+#include "sandboxed_api/tools/clang_generator/generator.h"
 #include "sandboxed_api/util/status_macros.h"
 
 namespace sapi {
@@ -106,6 +110,30 @@ constexpr absl::string_view kClassFooterTemplate = R"(
 };
 )";
 
+namespace internal {
+
+absl::StatusOr<std::string> ReformatGoogleStyle(const std::string& filename,
+                                                const std::string& code) {
+  // Configure code style based on Google style, but enforce pointer alignment
+  clang::format::FormatStyle style =
+      clang::format::getGoogleStyle(clang::format::FormatStyle::LK_Cpp);
+  style.DerivePointerAlignment = false;
+  style.PointerAlignment = clang::format::FormatStyle::PAS_Left;
+
+  clang::tooling::Replacements replacements = clang::format::reformat(
+      style, code, llvm::makeArrayRef(clang::tooling::Range(0, code.size())),
+      filename);
+
+  llvm::Expected<std::string> formatted_header =
+      clang::tooling::applyAllReplacements(code, replacements);
+  if (!formatted_header) {
+    return absl::InternalError(llvm::toString(formatted_header.takeError()));
+  }
+  return *formatted_header;
+}
+
+}  // namespace internal
+
 std::string GetIncludeGuard(absl::string_view filename) {
   if (filename.empty()) {
     static auto* bit_gen = new absl::BitGen();
@@ -250,7 +278,8 @@ absl::StatusOr<std::string> EmitFunction(const clang::FunctionDecl* decl) {
 }
 
 absl::StatusOr<std::string> EmitHeader(
-    std::vector<clang::FunctionDecl*> functions, const QualTypeSet& types,
+    const std::vector<std::string>& functions,
+    const Emitter::RenderedTypesMap& rendered_types,
     const GeneratorOptions& options) {
   std::string out;
   const std::string include_guard = GetIncludeGuard(options.out_file);
@@ -276,45 +305,19 @@ absl::StatusOr<std::string> EmitHeader(
   }
 
   // Emit type dependencies
-  // TODO(cblichmann): Coalesce namespaces
+  if (!rendered_types.empty()) {
-  std::string out_types = "// Types this API depends on\n";
+    absl::StrAppend(&out, "// Types this API depends on\n");
-  bool added_types = false;
+    for (const auto& [ns_name, types] : rendered_types) {
-  for (const clang::QualType& qual : types) {
+      if (!ns_name.empty()) {
-    clang::TypeDecl* decl = nullptr;
+        absl::StrAppend(&out, "namespace ", ns_name, " {\n");
-    if (const auto* typedef_type = qual->getAs<clang::TypedefType>()) {
+      }
-      decl = typedef_type->getDecl();
+      for (const auto& type : types) {
-    } else if (const auto* enum_type = qual->getAs<clang::EnumType>()) {
+        absl::StrAppend(&out, type, ";\n");
-      decl = enum_type->getDecl();
+      }
-    } else {
+      if (!ns_name.empty()) {
-      decl = qual->getAsRecordDecl();
+        absl::StrAppend(&out, "}  // namespace ", ns_name, "\n\n");
-    }
-    if (!decl) {
-      continue;
-    }
-
-    const std::vector<std::string> ns_path = GetNamespacePath(decl);
-    std::string nested_ns_name;
-    if (!ns_path.empty()) {
-      if (const auto& ns_root = ns_path.front();
-          ns_root == "std" || ns_root == "sapi" || ns_root == "__gnu_cxx") {
-        // Filter out any and all declarations from the C++ standard library,
-        // from SAPI itself and from other well-known namespaces. This avoids
-        // re-declaring things like standard integer types, for example.
-        continue;
       }
-      nested_ns_name = absl::StrCat(ns_path[0].empty() ? "" : " ",
-                                    absl::StrJoin(ns_path, "::"));
-      absl::StrAppend(&out_types, "namespace", nested_ns_name, " {\n");
     }
-    absl::StrAppend(&out_types, PrintAstDecl(decl), ";");
-    if (!ns_path.empty()) {
-      absl::StrAppend(&out_types, "\n}  // namespace", nested_ns_name);
-    }
-    absl::StrAppend(&out_types, "\n");
-    added_types = true;
-  }
-  if (added_types) {
-    absl::StrAppend(&out, out_types);
   }
 
   // Optionally emit a default sandbox that instantiates an embedded sandboxee
@@ -329,11 +332,7 @@ absl::StatusOr<std::string> EmitHeader(
   // TODO(cblichmann): Make the "Api" suffix configurable or at least optional.
   absl::StrAppendFormat(&out, kClassHeaderTemplate,
                         absl::StrCat(options.name, "Api"));
-  std::string out_func;
+  absl::StrAppend(&out, absl::StrJoin(functions, "\n"));
-  for (const clang::FunctionDecl* decl : functions) {
-    SAPI_ASSIGN_OR_RETURN(out_func, EmitFunction(decl));
-    absl::StrAppend(&out, out_func);
-  }
   absl::StrAppend(&out, kClassFooterTemplate);
 
   // Close out the header: close namespace (if needed) and end include guard
@@ -344,4 +343,45 @@ absl::StatusOr<std::string> EmitHeader(
   return out;
 }
 
+void Emitter::CollectType(clang::QualType qual) {
+  clang::TypeDecl* decl = nullptr;
+  if (const auto* typedef_type = qual->getAs<clang::TypedefType>()) {
+    decl = typedef_type->getDecl();
+  } else if (const auto* enum_type = qual->getAs<clang::EnumType>()) {
+    decl = enum_type->getDecl();
+  } else {
+    decl = qual->getAsRecordDecl();
+  }
+  if (!decl) {
+    return;
+  }
+
+  const std::vector<std::string> ns_path = GetNamespacePath(decl);
+  std::string ns_name;
+  if (!ns_path.empty()) {
+    if (const auto& ns_root = ns_path.front();
+        ns_root == "std" || ns_root == "sapi" || ns_root == "__gnu_cxx") {
+      // Filter out any and all declarations from the C++ standard library,
+      // from SAPI itself and from other well-known namespaces. This avoids
+      // re-declaring things like standard integer types, for example.
+      return;
+    }
+    ns_name = absl::StrCat(ns_path[0].empty() ? "" : " ",
+                           absl::StrJoin(ns_path, "::"));
+  }
+
+  rendered_types_[ns_name].push_back(PrintAstDecl(decl));
+}
+
+void Emitter::CollectFunction(clang::FunctionDecl* decl) {
+  functions_.push_back(*EmitFunction(decl));  // Cannot currently fail
+}
+
+absl::StatusOr<std::string> Emitter::EmitHeader(
+    const GeneratorOptions& options) {
+  SAPI_ASSIGN_OR_RETURN(const std::string header,
+                   ::sapi::EmitHeader(functions_, rendered_types_, options));
+  return internal::ReformatGoogleStyle(options.out_file, header);
+}
+
 }  // namespace sapi

sandboxed_api/tools/clang_generator/emitter.h

@@ -16,16 +16,44 @@
 #define SANDBOXED_API_TOOLS_CLANG_GENERATOR_EMITTER_H_
 
 #include <string>
+#include <vector>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/string_view.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Type.h"
-#include "sandboxed_api/tools/clang_generator/generator.h"
 #include "sandboxed_api/tools/clang_generator/types.h"
 
 namespace sapi {
+namespace internal {
+
+absl::StatusOr<std::string> ReformatGoogleStyle(const std::string& filename,
+                                                const std::string& code);
+
+}  // namespace internal
+
+class GeneratorOptions;
+
+class Emitter {
+ public:
+  using RenderedTypesMap =
+      absl::flat_hash_map<std::string, std::vector<std::string>>;
+
+  void CollectType(clang::QualType qual);
+  void CollectFunction(clang::FunctionDecl* decl);
+
+  // Outputs a formatted header for a list of functions and their related types.
+  absl::StatusOr<std::string> EmitHeader(const GeneratorOptions& options);
+
+ protected:
+  // Maps namespace to a list of spellings for types
+  RenderedTypesMap rendered_types_;
+
+  // Functions for sandboxed API, including their bodies
+  std::vector<std::string> functions_;
+};
 
 // Constructs an include guard name for the given filename. The name is of the
 // same form as the include guards in this project.
@@ -35,11 +63,6 @@ namespace sapi {
 //   SANDBOXED_API_EXAMPLES_ZLIB_ZLIB_SAPI_SAPI_H_
 std::string GetIncludeGuard(absl::string_view filename);
 
-// Outputs a formatted header for a list of functions and their related types.
-absl::StatusOr<std::string> EmitHeader(
-    std::vector<clang::FunctionDecl*> functions, const QualTypeSet& types,
-    const GeneratorOptions& options);
-
 }  // namespace sapi
 
 #endif  // SANDBOXED_API_TOOLS_CLANG_GENERATOR_EMITTER_H_

sandboxed_api/tools/clang_generator/emitter_test.cc

@@ -14,18 +14,48 @@
 
 #include "sandboxed_api/tools/clang_generator/emitter.h"
 
+#include <initializer_list>
+
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include "absl/memory/memory.h"
 #include "absl/strings/string_view.h"
+#include "sandboxed_api/sandbox2/testing.h"
+#include "sandboxed_api/tools/clang_generator/frontend_action_test_util.h"
+#include "sandboxed_api/tools/clang_generator/generator.h"
 #include "sandboxed_api/util/status_matchers.h"
 
 namespace sapi {
 namespace {
 
 using ::testing::MatchesRegex;
+using ::testing::SizeIs;
 using ::testing::StrEq;
 using ::testing::StrNe;
 
+class EmitterForTesting : public Emitter {
+ public:
+  using Emitter::functions_;
+};
+
+class EmitterTest : public FrontendActionTest {};
+
+TEST_F(EmitterTest, BasicFunctionality) {
+  GeneratorOptions options;
+  options.out_file = "input.h";
+  options.set_function_names<std::initializer_list<std::string>>(
+      {"ExposedFunction"});
+
+  EmitterForTesting emitter;
+  RunFrontendAction(R"(extern "C" void ExposedFunction() {})",
+                    absl::make_unique<GeneratorAction>(emitter, options));
+
+  EXPECT_THAT(emitter.functions_, SizeIs(1));
+
+  absl::StatusOr<std::string> header = emitter.EmitHeader(options);
+  EXPECT_THAT(header, IsOk());
+}
+
 TEST(IncludeGuard, CreatesRandomizedGuardForEmptyFilename) {
   // Copybara will transform the string. This is intentional.
   constexpr absl::string_view kGeneratedHeaderPrefix =

sandboxed_api/tools/clang_generator/frontend_action_test_util.cc

@@ -0,0 +1,70 @@
+// Copyright 2020 google llc
+//
+// licensed under the apache license, version 2.0 (the "license");
+// you may not use this file except in compliance with the license.
+// you may obtain a copy of the license at
+//
+//     http://www.apache.org/licenses/license-2.0
+//
+// unless required by applicable law or agreed to in writing, software
+// distributed under the license is distributed on an "as is" basis,
+// without warranties or conditions of any kind, either express or implied.
+// see the license for the specific language governing permissions and
+// limitations under the license.
+
+#include "sandboxed_api/tools/clang_generator/frontend_action_test_util.h"
+
+#include <vector>
+
+#include "absl/status/status.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemOptions.h"
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+namespace sapi {
+namespace internal {
+
+absl::Status RunClangTool(
+    const std::vector<std::string> command_line,
+    const absl::flat_hash_map<std::string, std::string> file_contents,
+    std::unique_ptr<clang::FrontendAction> action) {
+  // Setup an in-memory virtual filesystem
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> fs(
+      new llvm::vfs::InMemoryFileSystem());
+  llvm::IntrusiveRefCntPtr<clang::FileManager> files =
+      new clang::FileManager(clang::FileSystemOptions(), fs);
+
+  for (const auto& [filename, content] : file_contents) {
+    if (!fs->addFile(filename, /*ModificationTime=*/0,
+                     llvm::MemoryBuffer::getMemBuffer(content))) {
+      return absl::UnknownError(
+          absl::StrCat("Couldn't add file to in-memory VFS: ", filename));
+    }
+  }
+
+#if LLVM_VERSION_MAJOR >= 10
+  clang::tooling::ToolInvocation invocation(command_line, std::move(action),
+                                            files.get());
+#else
+  clang::tooling::ToolInvocation invocation(command_line, action.get(),
+                                            files.get());
+#endif
+  if (!invocation.run()) {
+    return absl::UnknownError("Tool invocation failed");
+  }
+  return absl::OkStatus();
+}
+
+}  // namespace internal
+
+std::vector<std::string> FrontendActionTest::GetCommandLineFlagsForTesting(
+    absl::string_view input_file) {
+  return {"tool", "-fsyntax-only", "--std=c++17",
+          "-I.",  "-Wno-error",    std::string(input_file)};
+}
+
+}  // namespace sapi

sandboxed_api/tools/clang_generator/frontend_action_test_util.h

@@ -0,0 +1,83 @@
+// Copyright 2020 google llc
+//
+// licensed under the apache license, version 2.0 (the "license");
+// you may not use this file except in compliance with the license.
+// you may obtain a copy of the license at
+//
+//     http://www.apache.org/licenses/license-2.0
+//
+// unless required by applicable law or agreed to in writing, software
+// distributed under the license is distributed on an "as is" basis,
+// without warranties or conditions of any kind, either express or implied.
+// see the license for the specific language governing permissions and
+// limitations under the license.
+
+#ifndef SANDBOXED_API_TOOLS_CLANG_GENERATOR_FRONTEND_ACTION_TEST_UTIL_H_
+#define SANDBOXED_API_TOOLS_CLANG_GENERATOR_FRONTEND_ACTION_TEST_UTIL_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "absl/container/flat_hash_map.h"
+#include "absl/memory/memory.h"
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "absl/strings/string_view.h"
+#include "clang/Frontend/FrontendAction.h"
+#include "sandboxed_api/util/status_matchers.h"
+
+namespace sapi {
+namespace internal {
+
+absl::Status RunClangTool(
+    const std::vector<std::string> command_line,
+    const absl::flat_hash_map<std::string, std::string> file_contents,
+    std::unique_ptr<clang::FrontendAction> action);
+
+}  // namespace internal
+
+class FrontendActionTest : public ::testing::Test {
+ protected:
+  // Adds code to a virtual filesystem with the given filename.
+  void AddCode(const std::string& filename, absl::string_view code) {
+    absl::StrAppend(&file_contents_[filename], code);
+  }
+
+  // Changes the name of the virtual input file. Useful for special cases where
+  // the filenames of compiled sources matter.
+  void set_input_file(absl::string_view value) {
+    input_file_ = std::string(value);
+  }
+
+  virtual std::vector<std::string> GetCommandLineFlagsForTesting(
+      absl::string_view input_file);
+
+  // Runs the specified frontend action on in-memory source code.
+  void RunFrontendAction(absl::string_view code,
+                         std::unique_ptr<clang::FrontendAction> action) {
+    std::vector<std::string> command_line =
+        GetCommandLineFlagsForTesting(input_file_);
+    AddCode(input_file_, code);
+    ASSERT_THAT(
+        internal::RunClangTool(command_line, file_contents_, std::move(action)),
+        IsOk());
+  }
+
+  // Runs the specified frontend action. Provided for compatibility with LLVM <
+  // 10. Takes ownership.
+  void RunFrontendAction(absl::string_view code,
+                         clang::FrontendAction* action) {
+    RunFrontendAction(code, absl::WrapUnique(action));
+  }
+
+ private:
+  std::string input_file_ = "input.cc";
+  absl::flat_hash_map<std::string, std::string> file_contents_;
+};
+
+}  // namespace sapi
+
+#endif  // SANDBOXED_API_TOOLS_CLANG_GENERATOR_FRONTEND_ACTION_TEST_UTIL_H_

sandboxed_api/tools/clang_generator/generator.cc

@@ -19,6 +19,7 @@
 
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
+#include "clang/AST/Type.h"
 #include "clang/Format/Format.h"
 #include "sandboxed_api/sandbox2/util/fileops.h"
 #include "sandboxed_api/tools/clang_generator/diagnostics.h"
@@ -39,88 +40,46 @@ std::string ReplaceFileExtension(absl::string_view path,
   return absl::StrCat(path.substr(0, pos), new_extension);
 }
 
-std::string GetOutputFilename(absl::string_view source_file) {
-  return ReplaceFileExtension(source_file, ".sapi.h");
-}
-
 inline absl::string_view ToStringView(llvm::StringRef ref) {
   return absl::string_view(ref.data(), ref.size());
 }
 
 }  // namespace
 
+std::string GetOutputFilename(absl::string_view source_file) {
+  return ReplaceFileExtension(source_file, ".sapi.h");
+}
+
 bool GeneratorASTVisitor::VisitFunctionDecl(clang::FunctionDecl* decl) {
   if (!decl->isCXXClassMember() &&  // Skip classes
       decl->isExternC() &&          // Skip non external functions
       !decl->isTemplated() &&       // Skip function templates
       // Process either all function or just the requested ones
-      (options_->function_names.empty() ||
+      (options_.function_names.empty() ||
-       options_->function_names.count(ToStringView(decl->getName())) > 0)) {
+       options_.function_names.count(ToStringView(decl->getName())) > 0)) {
     functions_.push_back(decl);
-    GatherRelatedTypes(decl->getDeclaredReturnType(), &types_);
+
+    CollectRelatedTypes(decl->getDeclaredReturnType(), &types_);
     for (const clang::ParmVarDecl* param : decl->parameters()) {
-      GatherRelatedTypes(param->getType(), &types_);
+      CollectRelatedTypes(param->getType(), &types_);
     }
   }
   return true;
 }
 
-namespace internal {
-
-absl::StatusOr<std::string> ReformatGoogleStyle(const std::string& filename,
-                                                const std::string& code) {
-  // Configure code style based on Google style, but enforce pointer alignment
-  clang::format::FormatStyle style =
-      clang::format::getGoogleStyle(clang::format::FormatStyle::LK_Cpp);
-  style.DerivePointerAlignment = false;
-  style.PointerAlignment = clang::format::FormatStyle::PAS_Left;
-
-  clang::tooling::Replacements replacements = clang::format::reformat(
-      style, code, llvm::makeArrayRef(clang::tooling::Range(0, code.size())),
-      filename);
-
-  llvm::Expected<std::string> formatted_header =
-      clang::tooling::applyAllReplacements(code, replacements);
-  if (!formatted_header) {
-    return absl::InternalError(llvm::toString(formatted_header.takeError()));
-  }
-  return *formatted_header;
-}
-
-}  // namespace internal
-
-absl::Status GeneratorASTConsumer::GenerateAndSaveHeader() {
-  const std::string out_file =
-      options_->out_file.empty() ? GetOutputFilename(in_file_)
-                                 : sandbox2::file_util::fileops::MakeAbsolute(
-                                       options_->out_file, options_->work_dir);
-
-  SAPI_ASSIGN_OR_RETURN(const std::string header,
-                   EmitHeader(visitor_.functions_, visitor_.types_, *options_));
-  SAPI_ASSIGN_OR_RETURN(const std::string formatted_header,
-                   internal::ReformatGoogleStyle(in_file_, header));
-
-  std::ofstream os(out_file, std::ios::out | std::ios::trunc);
-  os << formatted_header;
-  if (!os) {
-    return absl::UnknownError("I/O error");
-  }
-  return absl::OkStatus();
-}
-
 void GeneratorASTConsumer::HandleTranslationUnit(clang::ASTContext& context) {
-  absl::Status status;
+  std::cout << "Processing " << in_file_ << "\n";
   if (!visitor_.TraverseDecl(context.getTranslationUnitDecl())) {
-    status = absl::InternalError("AST traversal exited early");
+    ReportFatalError(context.getDiagnostics(),
-  } else {
+                     context.getTranslationUnitDecl()->getBeginLoc(),
-    status = GenerateAndSaveHeader();
+                     "AST traversal exited early");
   }
 
-  if (!status.ok()) {
+  for (clang::QualType qual : visitor_.types_) {
-    ReportFatalError(context.getDiagnostics(),
+    emitter_.CollectType(qual);
-                     GetDiagnosticLocationFromStatus(status).value_or(
+  }
-                         context.getTranslationUnitDecl()->getBeginLoc()),
+  for (clang::FunctionDecl* func : visitor_.functions_) {
-                     status.message());
+    emitter_.CollectFunction(func);
   }
 }
 

sandboxed_api/tools/clang_generator/generator.h

@@ -26,6 +26,7 @@
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/Tooling/Tooling.h"
+#include "sandboxed_api/tools/clang_generator/emitter.h"
 #include "sandboxed_api/tools/clang_generator/types.h"
 
 namespace sapi {
@@ -46,85 +47,86 @@ struct GeneratorOptions {
   std::string work_dir;
   std::string name;            // Name of the Sandboxed API
   std::string namespace_name;  // Namespace to wrap the SAPI in
-  std::string out_file;        // Output path of the generated header
+  // Output path of the generated header. Used to build the header include
-  std::string embed_dir;       // Directory with embedded includes
+  // guard.
-  std::string embed_name;      // Identifier of the embed object
+  std::string out_file;
+  std::string embed_dir;   // Directory with embedded includes
+  std::string embed_name;  // Identifier of the embed object
 };
 
 class GeneratorASTVisitor
     : public clang::RecursiveASTVisitor<GeneratorASTVisitor> {
  public:
+  explicit GeneratorASTVisitor(const GeneratorOptions& options)
+      : options_(options) {}
+
   bool VisitFunctionDecl(clang::FunctionDecl* decl);
 
  private:
   friend class GeneratorASTConsumer;
-  const GeneratorOptions* options_ = nullptr;
 
   std::vector<clang::FunctionDecl*> functions_;
   QualTypeSet types_;
+
+  const GeneratorOptions& options_;
 };
 
-namespace internal {
-
-absl::StatusOr<std::string> ReformatGoogleStyle(const std::string& filename,
-                                                const std::string& code);
-
-}  // namespace internal
-
 class GeneratorASTConsumer : public clang::ASTConsumer {
  public:
-  GeneratorASTConsumer(std::string in_file, const GeneratorOptions* options)
+  GeneratorASTConsumer(std::string in_file, Emitter& emitter,
-      : in_file_(std::move(in_file)), options_(options) {
+                       const GeneratorOptions& options)
-    visitor_.options_ = options_;
+      : in_file_(std::move(in_file)), visitor_(options), emitter_(emitter) {}
-  }
 
  private:
   void HandleTranslationUnit(clang::ASTContext& context) override;
 
-  absl::Status GenerateAndSaveHeader();
-
   std::string in_file_;
-  const GeneratorOptions* options_;
-
   GeneratorASTVisitor visitor_;
+
+  Emitter& emitter_;
 };
 
 class GeneratorAction : public clang::ASTFrontendAction {
  public:
-  explicit GeneratorAction(const GeneratorOptions* options)
+  GeneratorAction(Emitter& emitter, const GeneratorOptions& options)
-      : options_(options) {}
+      : emitter_(emitter), options_(options) {}
 
  private:
   std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
       clang::CompilerInstance&, llvm::StringRef in_file) override {
     return absl::make_unique<GeneratorASTConsumer>(std::string(in_file),
-                                                   options_);
+                                                   emitter_, options_);
   }
 
   bool hasCodeCompletionSupport() const override { return false; }
 
-  const GeneratorOptions* options_;
+  Emitter& emitter_;
+  const GeneratorOptions& options_;
 };
 
 class GeneratorFactory : public clang::tooling::FrontendActionFactory {
  public:
-  explicit GeneratorFactory(GeneratorOptions options = {})
+  // Does not take ownership
-      : options_(std::move(options)) {}
+  GeneratorFactory(Emitter& emitter, const GeneratorOptions& options)
+      : emitter_(emitter), options_(options) {}
 
  private:
 #if LLVM_VERSION_MAJOR >= 10
   std::unique_ptr<clang::FrontendAction> create() override {
-    return absl::make_unique<GeneratorAction>(&options_);
+    return absl::make_unique<GeneratorAction>(emitter_, options_);
   }
 #else
   clang::FrontendAction* create() override {
-    return new GeneratorAction(&options_);
+    return new GeneratorAction(emitter_, options_);
   }
 #endif
 
-  GeneratorOptions options_;
+  Emitter& emitter_;
+  const GeneratorOptions& options_;
 };
 
+std::string GetOutputFilename(absl::string_view source_file);
+
 }  // namespace sapi
 
 #endif  // SANDBOXED_API_TOOLS_CLANG_GENERATOR_GENERATOR_H_

sandboxed_api/tools/clang_generator/generator_tool.cc

@@ -12,16 +12,23 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstdlib>
 #include <string>
 #include <vector>
 
 #include "absl/memory/memory.h"
+#include "absl/status/status.h"
+#include "absl/strings/str_format.h"
+#include "absl/strings/str_split.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/ArgumentsAdjusters.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "llvm/Support/CommandLine.h"
+#include "sandboxed_api/sandbox2/util/file_helpers.h"
 #include "sandboxed_api/sandbox2/util/fileops.h"
 #include "sandboxed_api/tools/clang_generator/generator.h"
+#include "sandboxed_api/util/status_macros.h"
 
 namespace sapi {
 namespace {
@@ -49,11 +56,11 @@ static auto* g_sapi_functions = new llvm::cl::list<std::string>(
     llvm::cl::cat(*g_tool_category));
 static auto* g_sapi_in = new llvm::cl::list<std::string>(
     "sapi_in", llvm::cl::CommaSeparated,
-    llvm::cl::desc("List of input files to analyze. Deprecated, use positional "
+    llvm::cl::desc("List of input files to analyze."),
-                   "arguments instead."),
     llvm::cl::cat(*g_tool_category));
 static auto* g_sapi_isystem = new llvm::cl::opt<std::string>(
-    "sapi_isystem", llvm::cl::desc("Extra system include paths"),
+    "sapi_isystem",
+    llvm::cl::desc("Parameter file with extra system include paths"),
     llvm::cl::cat(*g_tool_category));
 static auto* g_sapi_limit_scan_depth = new llvm::cl::opt<bool>(
     "sapi_limit_scan_depth",
@@ -69,28 +76,28 @@ static auto* g_sapi_ns = new llvm::cl::opt<std::string>(
 static auto* g_sapi_out = new llvm::cl::opt<std::string>(
     "sapi_out",
     llvm::cl::desc(
-        "Ouput path of the generated header. If empty, simply appends .sapi.h "
+        "Output path of the generated header. If empty, simply appends .sapi.h "
         "to the basename of the first source file specified."),
     llvm::cl::cat(*g_tool_category));
 
 }  // namespace
 
-GeneratorOptions GeneratorOptionsFromFlags() {
+GeneratorOptions GeneratorOptionsFromFlags(
+    const std::vector<std::string>& sources) {
   GeneratorOptions options;
   options.function_names.insert(g_sapi_functions->begin(),
                                 g_sapi_functions->end());
   options.work_dir = sandbox2::file_util::fileops::GetCWD();
   options.name = *g_sapi_name;
   options.namespace_name = *g_sapi_ns;
-  options.out_file = *g_sapi_out;
+  options.out_file =
+      !g_sapi_out->empty() ? *g_sapi_out : GetOutputFilename(sources.front());
   options.embed_dir = *g_sapi_embed_dir;
   options.embed_name = *g_sapi_embed_name;
   return options;
 }
 
-}  // namespace sapi
+absl::Status GeneratorMain(int argc, const char** argv) {
-
-int main(int argc, const char** argv) {
   clang::tooling::CommonOptionsParser opt_parser(
       argc, argv, *sapi::g_tool_category, llvm::cl::ZeroOrMore,
       "Generates a Sandboxed API header for C/C++ translation units.");
@@ -98,9 +105,45 @@ int main(int argc, const char** argv) {
   for (const auto& sapi_in : *sapi::g_sapi_in) {
     sources.push_back(sapi_in);
   }
+  if (sources.empty()) {
+    return absl::InvalidArgumentError("error: no input files");
+  }
+
+  auto options = sapi::GeneratorOptionsFromFlags(sources);
+  sapi::Emitter emitter;
 
   clang::tooling::ClangTool tool(opt_parser.getCompilations(), sources);
-  return tool.run(absl::make_unique<sapi::GeneratorFactory>(
+  if (!sapi::g_sapi_isystem->empty()) {
-                      sapi::GeneratorOptionsFromFlags())
+    std::string isystem_lines;
-                      .get());
+    SAPI_RETURN_IF_ERROR(sandbox2::file::GetContents(
+        *sapi::g_sapi_isystem, &isystem_lines, sandbox2::file::Defaults()));
+    std::vector<std::string> isystem =
+        absl::StrSplit(isystem_lines, '\n', absl::SkipWhitespace());
+    for (std::string& line : isystem) {
+      line.insert(0, "-isystem");
+    }
+    tool.appendArgumentsAdjuster(clang::tooling::getInsertArgumentAdjuster(
+        isystem, clang::tooling::ArgumentInsertPosition::BEGIN));
+  }
+  if (int result = tool.run(
+          absl::make_unique<sapi::GeneratorFactory>(emitter, options).get());
+      result != 0) {
+    return absl::UnknownError("header generation failed");
+  }
+
+  SAPI_ASSIGN_OR_RETURN(std::string header, emitter.EmitHeader(options));
+
+  SAPI_RETURN_IF_ERROR(sandbox2::file::SetContents(options.out_file, header,
+                                              sandbox2::file::Defaults()));
+  return absl::OkStatus();
+}
+
+}  // namespace sapi
+
+int main(int argc, const char** argv) {
+  if (absl::Status status = sapi::GeneratorMain(argc, argv); !status.ok()) {
+    absl::FPrintF(stderr, "error: %s\n", status.message());
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
 }

sandboxed_api/tools/clang_generator/types.cc

@@ -30,9 +30,9 @@ bool IsFunctionReferenceType(clang::QualType qual) {
 
 }  // namespace
 
-void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) {
+void CollectRelatedTypes(clang::QualType qual, QualTypeSet* types) {
   if (const auto* typedef_type = qual->getAs<clang::TypedefType>()) {
-    GatherRelatedTypes(typedef_type->getDecl()->getUnderlyingType(), types);
+    CollectRelatedTypes(typedef_type->getDecl()->getUnderlyingType(), types);
     types->insert(qual);
     return;
   }
@@ -43,9 +43,9 @@ void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) {
                                         ->getAs<clang::FunctionProtoType>()) {
       // Note: Do not add the function type itself, as this will always be a
       //       pointer argument. We only need to collect all its related types.
-      GatherRelatedTypes(function_type->getReturnType(), types);
+      CollectRelatedTypes(function_type->getReturnType(), types);
       for (const clang::QualType& param : function_type->getParamTypes()) {
-        GatherRelatedTypes(param, types);
+        CollectRelatedTypes(param, types);
       }
       return;
     }
@@ -56,13 +56,13 @@ void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) {
     while (IsPointerOrReference(pointee)) {
       pointee = pointee->getPointeeType();
     }
-    GatherRelatedTypes(pointee, types);
+    CollectRelatedTypes(pointee, types);
     return;
   }
 
   // C array with specified constant size (i.e. int a[42])?
   if (const clang::ArrayType* array_type = qual->getAsArrayTypeUnsafe()) {
-    GatherRelatedTypes(array_type->getElementType(), types);
+    CollectRelatedTypes(array_type->getElementType(), types);
     return;
   }
 
@@ -71,7 +71,7 @@ void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) {
       // Collect the underlying integer type of enum classes as well, as it may
       // be a typedef.
       if (const clang::EnumDecl* decl = enum_type->getDecl(); decl->isFixed()) {
-        GatherRelatedTypes(decl->getIntegerType(), types);
+        CollectRelatedTypes(decl->getIntegerType(), types);
       }
     }
     types->insert(qual);
@@ -81,7 +81,7 @@ void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types) {
   if (const auto* record_type = qual->getAs<clang::RecordType>()) {
     const clang::RecordDecl* decl = record_type->getDecl();
     for (const clang::FieldDecl* field : decl->fields()) {
-      GatherRelatedTypes(field->getType(), types);
+      CollectRelatedTypes(field->getType(), types);
     }
     types->insert(qual);
     return;

sandboxed_api/tools/clang_generator/types.h

@@ -54,7 +54,7 @@ inline bool IsPointerOrReference(clang::QualType qual) {
 //   int
 //   SubStruct
 //   bool
-void GatherRelatedTypes(clang::QualType qual, QualTypeSet* types);
+void CollectRelatedTypes(clang::QualType qual, QualTypeSet* types);
 
 // Maps a qualified type to a fully qualified SAPI-compatible type name. This
 // is used for the generated code that invokes the actual function call RPC.

sandboxed_api/tools/generator2/code.py

@@ -580,12 +580,11 @@ class _TranslationUnit(object):
 
       for i, cursor in enumerate(self._walk_preorder()):
         # Workaround for issue#32
+        # ignore all the cursors with kinds not implemented in python bindings
         try:
           cursor.kind
         except ValueError:
-          if cursor._kind_id >= 440:  # pylint: disable=protected-access
+          continue
-            continue
-          raise
         # naive way to order types: they should be ordered when walking the tree
         if cursor.kind.is_declaration():
           self.order[cursor.hash] = i
@@ -911,7 +910,7 @@ class Generator(object):
     result.append('#include "sandboxed_api/sandbox.h"')
     result.append('#include "sandboxed_api/vars.h"')
 
-    if (embed_dir is not None) and (embed_name is not None):
+    if embed_dir and embed_name:
       result.append(
           Generator.EMBED_INCLUDE.format(
               os.path.join(embed_dir, embed_name) + '_embed.h'))
@@ -928,7 +927,7 @@ class Generator(object):
 
     result.append('')
 
-    if (embed_dir is not None) and (embed_name is not None):
+    if embed_dir and embed_name:
       result.append(
           Generator.EMBED_CLASS.format(name, embed_name.replace('-', '_')))
 

sandboxed_api/util/BUILD.bazel

@@ -46,6 +46,7 @@ cc_library(
     name = "statusor",
     hdrs = ["statusor.h"],
     deprecation = "Migrate to `absl::StatusOr<T>`",
+    visibility = ["//visibility:public"],
     deps = [
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/status:statusor",

sandboxed_api/var_array.h

@@ -15,9 +15,11 @@
 #ifndef SANDBOXED_API_VAR_ARRAY_H_
 #define SANDBOXED_API_VAR_ARRAY_H_
 
+#include <algorithm>
 #include <cstring>
 #include <memory>
 
+#include <glog/logging.h>
 #include "absl/base/macros.h"
 #include "absl/status/status.h"
 #include "absl/strings/str_cat.h"
@@ -39,19 +41,19 @@ class Array : public Var, public Pointable {
         nelem_(nelem),
         total_size_(nelem_ * sizeof(T)),
         buffer_owned_(false) {
-    SetLocal(const_cast<void*>(reinterpret_cast<const void*>(arr_)));
+    SetLocal(const_cast<std::remove_const_t<T>*>(arr_));
   }
   // The array is allocated and owned by this object.
   explicit Array(size_t nelem)
-      : arr_(static_cast<T*>(malloc(sizeof(T) * nelem))),
+      : nelem_(nelem), total_size_(nelem_ * sizeof(T)), buffer_owned_(true) {
-        nelem_(nelem),
+    void* storage = malloc(sizeof(T) * nelem);
-        total_size_(nelem_ * sizeof(T)),
+    CHECK(storage != nullptr);
-        buffer_owned_(true) {
+    SetLocal(storage);
-    SetLocal(const_cast<void*>(reinterpret_cast<const void*>(arr_)));
+    arr_ = static_cast<T*>(storage);
   }
   virtual ~Array() {
     if (buffer_owned_) {
-      free(const_cast<void*>(reinterpret_cast<const void*>(arr_)));
+      free(const_cast<std::remove_const_t<T>*>(arr_));
     }
   }
 
@@ -121,7 +123,7 @@ class Array : public Var, public Pointable {
     arr_ = static_cast<T*>(new_addr);
     total_size_ = size;
     nelem_ = size / sizeof(T);
-    SetLocal(arr_);
+    SetLocal(new_addr);
     return absl::OkStatus();
   }
 
@@ -142,7 +144,7 @@ class Array : public Var, public Pointable {
 class CStr : public Array<char> {
  public:
   explicit CStr(char* cstr) : Array<char>(strlen(cstr) + 1) {
-    strcpy(this->GetData(), cstr);  // NOLINT
+    std::copy(cstr, cstr + GetNElem(), GetData());
   }
 
   std::string ToString() const final {
@@ -159,10 +161,7 @@ class ConstCStr : public Array<const char> {
       : Array<const char>(cstr, strlen(cstr) + 1) {}
 
   std::string ToString() const final {
-    if (GetData() == nullptr) {
+    return absl::StrCat("ConstCStr: len(w/o NUL):", strlen(GetData()), ", ['",
-      return "CStr: [nullptr]";
-    }
-    return absl::StrCat("CStr: len(w/o NUL):", strlen(GetData()), ", ['",
                         GetData(), "']");
   }
 };