From 78b8e130d7fb00de648489692be9a3116177c8f1 Mon Sep 17 00:00:00 2001 From: Katarzyna Miernikiewicz Date: Tue, 8 Sep 2020 10:37:17 +0000 Subject: [PATCH 01/28] first jsonnet example --- .gitmodules | 3 + oss-internship-2020/jsonnet/CMakeLists.txt | 63 ++++++++++ .../jsonnet/examples/CMakeLists.txt | 16 +++ .../jsonnet/examples/jsonnet_example.cc | 108 ++++++++++++++++++ oss-internship-2020/jsonnet/jsonnet | 1 + oss-internship-2020/jsonnet/jsonnet_helper.cc | 51 +++++++++ oss-internship-2020/jsonnet/jsonnet_helper.h | 33 ++++++ 7 files changed, 275 insertions(+) create mode 100644 .gitmodules create mode 100644 oss-internship-2020/jsonnet/CMakeLists.txt create mode 100644 oss-internship-2020/jsonnet/examples/CMakeLists.txt create mode 100644 oss-internship-2020/jsonnet/examples/jsonnet_example.cc create mode 160000 oss-internship-2020/jsonnet/jsonnet create mode 100644 oss-internship-2020/jsonnet/jsonnet_helper.cc create mode 100644 oss-internship-2020/jsonnet/jsonnet_helper.h diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..12df9a5 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "oss-internship-2020/jsonnet/jsonnet"] + path = oss-internship-2020/jsonnet/jsonnet + url = https://github.com/google/jsonnet.git diff --git a/oss-internship-2020/jsonnet/CMakeLists.txt b/oss-internship-2020/jsonnet/CMakeLists.txt new file mode 100644 index 0000000..37b05b9 --- /dev/null +++ b/oss-internship-2020/jsonnet/CMakeLists.txt @@ -0,0 +1,63 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.10) + +project(jsonnet-sapi C CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +add_subdirectory(jsonnet) + +add_library(jsonnet_helper STATIC + jsonnet_helper.cc + jsonnet_helper.h + ${CMAKE_CURRENT_SOURCE_DIR}/jsonnet/cmd/utils.h + ${CMAKE_CURRENT_SOURCE_DIR}/jsonnet/cmd/utils.cpp +) + +target_link_libraries(jsonnet_helper + libjsonnet_for_binaries +) + +set(SAPI_ROOT "/usr/local/google/home/kasiamm/sandboxed-api" CACHE PATH "Path to the Sandboxed API source tree") +set(SAPI_ENABLE_EXAMPLES OFF CACHE BOOL "") +set(SAPI_ENABLE_TESTS OFF CACHE BOOL "") + +add_subdirectory("${SAPI_ROOT}" + "${CMAKE_BINARY_DIR}/sandboxed-api-build" + EXCLUDE_FROM_ALL) + +add_sapi_library(jsonnet_sapi + FUNCTIONS c_jsonnet_evaluate_snippet + c_jsonnet_make + c_jsonnet_destroy + c_read_input + c_free_input + c_write_output_file + c_jsonnet_realloc + INPUTS jsonnet_helper.h + LIBRARY jsonnet_helper + LIBRARY_NAME Jsonnet + NAMESPACE "" +) + +target_include_directories(jsonnet_sapi INTERFACE + "${PROJECT_BINARY_DIR}" +) + +target_link_libraries(jsonnet_sapi PUBLIC jsonnet_helper) + +add_subdirectory(examples) diff --git a/oss-internship-2020/jsonnet/examples/CMakeLists.txt b/oss-internship-2020/jsonnet/examples/CMakeLists.txt new file mode 100644 index 0000000..6cbc323 --- /dev/null +++ b/oss-internship-2020/jsonnet/examples/CMakeLists.txt @@ -0,0 +1,16 @@ + +#target_include_directories(jsonnet_helper PUBLIC +# "${CMAKE_CURRENT_SOURCE_DIR}/jsonnet" +# "${PROJECT_BINARY_DIR}" +#) + +add_executable(jsonnet_sandboxed + jsonnet_example.cc +) + +target_link_libraries(jsonnet_sandboxed PRIVATE + libjsonnet + jsonnet_helper + jsonnet_sapi + sapi::sapi +) diff --git a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc new file mode 100644 index 0000000..2ace227 --- /dev/null +++ b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc @@ -0,0 +1,108 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include + +#include "jsonnet_sapi.sapi.h" +#include "sandboxed_api/util/flag.h" + +class JsonnetSapiSandbox : public JsonnetSandbox { + public: + explicit JsonnetSapiSandbox(std::string in_file, std::string out_file) + : in_file_(std::move(in_file)), out_file_(std::move(out_file)) {} + // explicit JsonnetSapiSandbox() {}; + + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder*) override { + return sandbox2::PolicyBuilder() + .AllowStaticStartup() + .AllowOpen() + .AllowRead() + .AllowWrite() + .AllowStat() + .AllowSystemMalloc() + .AllowExit() + .AllowSyscalls({ + __NR_futex, + __NR_close, + }) + .AddFile(in_file_) + .AddDirectoryAt(dirname(&out_file_[0]), "/output", /*is_ro=*/false) + .BuildOrDie(); + } + private: + std::string in_file_; + std::string out_file_; + +}; + +int main(int argc, char* argv[]) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + + if (argc != 3) { + std::cerr << "Usage: " << basename(argv[0]) << " absolute/path/to/INPUT.jsonnet" + << " absolute/path/to/OUTPUT\n"; + return EXIT_FAILURE; + } + + std::string in_file(argv[1]); + std::string out_file(argv[2]); + + // Initialize sandbox. + JsonnetSapiSandbox sandbox(in_file, out_file); + absl::Status status = sandbox.Init(); + CHECK(status.ok()) << "Sandbox initialization failed " << status; + + JsonnetApi api(&sandbox); + + // Initialize library's main structure. + sapi::StatusOr jsonnet_vm = api.c_jsonnet_make(); + sapi::v::RemotePtr vm_pointer(jsonnet_vm.value()); + CHECK(jsonnet_vm.ok()) << "JsonnetVm initialization failed: " << jsonnet_vm.status(); + + // Read input file. + sapi::v::ConstCStr in_file_var(in_file.c_str()); + sapi::StatusOr input = api.c_read_input(false, in_file_var.PtrBefore()); + CHECK(input.ok()) << "Reading input file failed " << input.status(); + + // Process jsonnet data. + sapi::v::RemotePtr input_pointer(input.value()); + sapi::v::Int error; + sapi::StatusOr output = api.c_jsonnet_evaluate_snippet(&vm_pointer, in_file_var.PtrBefore(), &input_pointer, error.PtrAfter()); + CHECK(output.ok() && !error.GetValue()) << "Jsonnet code evaluation failed" << output.status() << " " << error.GetValue(); + + // Write data to file. + std::string out_file_in_sandboxee(std::string("/output/") + basename(&out_file[0])); + sapi::v::ConstCStr out_file_var(out_file_in_sandboxee.c_str()); + sapi::v::RemotePtr output_pointer(output.value()); + sapi::StatusOr success = api.c_write_output_file(&output_pointer, out_file_var.PtrBefore()); + CHECK(success.ok() && success.value()) << "Writing to output file failed " << success.status() << " " << success.value(); + + // Clean up. + sapi::StatusOr result = api.c_jsonnet_realloc(&vm_pointer, &output_pointer, 0); + CHECK(result.ok()) << "JsonnetVm realloc failed " << result.status(); + + status = api.c_jsonnet_destroy(&vm_pointer); + CHECK(status.ok()) << "JsonnetVm destroy failed " << status; + + status = api.c_free_input(&input_pointer); + CHECK(status.ok()) << "Input freeing failed " << status; + + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/oss-internship-2020/jsonnet/jsonnet b/oss-internship-2020/jsonnet/jsonnet new file mode 160000 index 0000000..3e25595 --- /dev/null +++ b/oss-internship-2020/jsonnet/jsonnet @@ -0,0 +1 @@ +Subproject commit 3e25595d5c4acd32a1c3951a57471986b90d3bad diff --git a/oss-internship-2020/jsonnet/jsonnet_helper.cc b/oss-internship-2020/jsonnet/jsonnet_helper.cc new file mode 100644 index 0000000..2b0270e --- /dev/null +++ b/oss-internship-2020/jsonnet/jsonnet_helper.cc @@ -0,0 +1,51 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jsonnet_helper.h" +#include + +struct JsonnetVm* c_jsonnet_make(void) { + return jsonnet_make(); +} + +void c_jsonnet_destroy(struct JsonnetVm* vm) { + return jsonnet_destroy(vm); +} + +char* c_jsonnet_evaluate_snippet(struct JsonnetVm* vm, const char* filename, char* snippet, int* error) { + return jsonnet_evaluate_snippet(vm, filename, snippet, error); +} + +char* c_read_input(bool filename_is_code, const char* filename) { + std::string s_filename(filename); + std::string s_input; + bool check = read_input(filename_is_code, &s_filename, &s_input); + char* c_input = strdup(s_input.c_str()); + if (check) return c_input; + return nullptr; +} + +void c_free_input(char* input) { + free(input); + return; +} + +bool c_write_output_file(const char* output, const char* output_file) { + std::string s_output_file(output_file); + return write_output_file(output, s_output_file); +} + +char* c_jsonnet_realloc(JsonnetVm* vm, char* str, size_t sz) { + return jsonnet_realloc(vm, str, sz); +} \ No newline at end of file diff --git a/oss-internship-2020/jsonnet/jsonnet_helper.h b/oss-internship-2020/jsonnet/jsonnet_helper.h new file mode 100644 index 0000000..a586ac0 --- /dev/null +++ b/oss-internship-2020/jsonnet/jsonnet_helper.h @@ -0,0 +1,33 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "jsonnet/cmd/utils.h" + +extern "C" { +#include +} + +extern "C" struct JsonnetVm* c_jsonnet_make(void); + +extern "C" void c_jsonnet_destroy(struct JsonnetVm* vm); + +extern "C" char* c_jsonnet_evaluate_snippet(struct JsonnetVm* vm, const char* filename, char* snippet, int* error); + +extern "C" char* c_read_input(bool filename_is_code, const char* filename); + +extern "C" void c_free_input(char* input); + +extern "C" bool c_write_output_file(const char* output, const char* output_file); + +extern "C" char* c_jsonnet_realloc(JsonnetVm* vm, char* str, size_t sz); \ No newline at end of file From 9f0bdf1c6384ead5d5417379b14a98f11e7fb134 Mon Sep 17 00:00:00 2001 From: Katarzyna Miernikiewicz Date: Tue, 8 Sep 2020 11:03:02 +0000 Subject: [PATCH 02/28] licence added --- oss-internship-2020/jsonnet/CMakeLists.txt | 2 +- .../jsonnet/examples/CMakeLists.txt | 18 +++++++++++++----- .../jsonnet/examples/jsonnet_example.cc | 1 - 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/oss-internship-2020/jsonnet/CMakeLists.txt b/oss-internship-2020/jsonnet/CMakeLists.txt index 37b05b9..9054beb 100644 --- a/oss-internship-2020/jsonnet/CMakeLists.txt +++ b/oss-internship-2020/jsonnet/CMakeLists.txt @@ -32,7 +32,7 @@ target_link_libraries(jsonnet_helper libjsonnet_for_binaries ) -set(SAPI_ROOT "/usr/local/google/home/kasiamm/sandboxed-api" CACHE PATH "Path to the Sandboxed API source tree") +set(SAPI_ROOT "../.." CACHE PATH "Path to the Sandboxed API source tree") set(SAPI_ENABLE_EXAMPLES OFF CACHE BOOL "") set(SAPI_ENABLE_TESTS OFF CACHE BOOL "") diff --git a/oss-internship-2020/jsonnet/examples/CMakeLists.txt b/oss-internship-2020/jsonnet/examples/CMakeLists.txt index 6cbc323..be008f5 100644 --- a/oss-internship-2020/jsonnet/examples/CMakeLists.txt +++ b/oss-internship-2020/jsonnet/examples/CMakeLists.txt @@ -1,8 +1,16 @@ - -#target_include_directories(jsonnet_helper PUBLIC -# "${CMAKE_CURRENT_SOURCE_DIR}/jsonnet" -# "${PROJECT_BINARY_DIR}" -#) +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. add_executable(jsonnet_sandboxed jsonnet_example.cc diff --git a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc index 2ace227..30fc3e3 100644 --- a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc +++ b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc @@ -25,7 +25,6 @@ class JsonnetSapiSandbox : public JsonnetSandbox { public: explicit JsonnetSapiSandbox(std::string in_file, std::string out_file) : in_file_(std::move(in_file)), out_file_(std::move(out_file)) {} - // explicit JsonnetSapiSandbox() {}; std::unique_ptr ModifyPolicy( sandbox2::PolicyBuilder*) override { From c8da7c3db559cea7fd8d82b38fe63b5ef0a187f2 Mon Sep 17 00:00:00 2001 From: Katarzyna Miernikiewicz Date: Tue, 8 Sep 2020 11:11:08 +0000 Subject: [PATCH 03/28] Google style applied --- .../jsonnet/examples/jsonnet_example.cc | 101 ++++++++++-------- oss-internship-2020/jsonnet/jsonnet_helper.cc | 34 +++--- oss-internship-2020/jsonnet/jsonnet_helper.h | 7 +- 3 files changed, 78 insertions(+), 64 deletions(-) diff --git a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc index 30fc3e3..04e116c 100644 --- a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc +++ b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include +#include #include #include @@ -27,7 +27,7 @@ class JsonnetSapiSandbox : public JsonnetSandbox { : in_file_(std::move(in_file)), out_file_(std::move(out_file)) {} std::unique_ptr ModifyPolicy( - sandbox2::PolicyBuilder*) override { + sandbox2::PolicyBuilder *) override { return sandbox2::PolicyBuilder() .AllowStaticStartup() .AllowOpen() @@ -44,64 +44,75 @@ class JsonnetSapiSandbox : public JsonnetSandbox { .AddDirectoryAt(dirname(&out_file_[0]), "/output", /*is_ro=*/false) .BuildOrDie(); } - private: - std::string in_file_; - std::string out_file_; + private: + std::string in_file_; + std::string out_file_; }; -int main(int argc, char* argv[]) { - google::InitGoogleLogging(argv[0]); - gflags::ParseCommandLineFlags(&argc, &argv, true); +int main(int argc, char *argv[]) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); - if (argc != 3) { - std::cerr << "Usage: " << basename(argv[0]) << " absolute/path/to/INPUT.jsonnet" + if (argc != 3) { + std::cerr << "Usage: " << basename(argv[0]) + << " absolute/path/to/INPUT.jsonnet" << " absolute/path/to/OUTPUT\n"; return EXIT_FAILURE; - } + } - std::string in_file(argv[1]); - std::string out_file(argv[2]); + std::string in_file(argv[1]); + std::string out_file(argv[2]); - // Initialize sandbox. - JsonnetSapiSandbox sandbox(in_file, out_file); - absl::Status status = sandbox.Init(); - CHECK(status.ok()) << "Sandbox initialization failed " << status; + // Initialize sandbox. + JsonnetSapiSandbox sandbox(in_file, out_file); + absl::Status status = sandbox.Init(); + CHECK(status.ok()) << "Sandbox initialization failed " << status; - JsonnetApi api(&sandbox); + JsonnetApi api(&sandbox); - // Initialize library's main structure. - sapi::StatusOr jsonnet_vm = api.c_jsonnet_make(); - sapi::v::RemotePtr vm_pointer(jsonnet_vm.value()); - CHECK(jsonnet_vm.ok()) << "JsonnetVm initialization failed: " << jsonnet_vm.status(); + // Initialize library's main structure. + sapi::StatusOr jsonnet_vm = api.c_jsonnet_make(); + sapi::v::RemotePtr vm_pointer(jsonnet_vm.value()); + CHECK(jsonnet_vm.ok()) << "JsonnetVm initialization failed: " + << jsonnet_vm.status(); - // Read input file. - sapi::v::ConstCStr in_file_var(in_file.c_str()); - sapi::StatusOr input = api.c_read_input(false, in_file_var.PtrBefore()); - CHECK(input.ok()) << "Reading input file failed " << input.status(); + // Read input file. + sapi::v::ConstCStr in_file_var(in_file.c_str()); + sapi::StatusOr input = + api.c_read_input(false, in_file_var.PtrBefore()); + CHECK(input.ok()) << "Reading input file failed " << input.status(); - // Process jsonnet data. - sapi::v::RemotePtr input_pointer(input.value()); - sapi::v::Int error; - sapi::StatusOr output = api.c_jsonnet_evaluate_snippet(&vm_pointer, in_file_var.PtrBefore(), &input_pointer, error.PtrAfter()); - CHECK(output.ok() && !error.GetValue()) << "Jsonnet code evaluation failed" << output.status() << " " << error.GetValue(); + // Process jsonnet data. + sapi::v::RemotePtr input_pointer(input.value()); + sapi::v::Int error; + sapi::StatusOr output = api.c_jsonnet_evaluate_snippet( + &vm_pointer, in_file_var.PtrBefore(), &input_pointer, error.PtrAfter()); + CHECK(output.ok() && !error.GetValue()) + << "Jsonnet code evaluation failed" << output.status() << " " + << error.GetValue(); - // Write data to file. - std::string out_file_in_sandboxee(std::string("/output/") + basename(&out_file[0])); - sapi::v::ConstCStr out_file_var(out_file_in_sandboxee.c_str()); - sapi::v::RemotePtr output_pointer(output.value()); - sapi::StatusOr success = api.c_write_output_file(&output_pointer, out_file_var.PtrBefore()); - CHECK(success.ok() && success.value()) << "Writing to output file failed " << success.status() << " " << success.value(); + // Write data to file. + std::string out_file_in_sandboxee(std::string("/output/") + + basename(&out_file[0])); + sapi::v::ConstCStr out_file_var(out_file_in_sandboxee.c_str()); + sapi::v::RemotePtr output_pointer(output.value()); + sapi::StatusOr success = + api.c_write_output_file(&output_pointer, out_file_var.PtrBefore()); + CHECK(success.ok() && success.value()) + << "Writing to output file failed " << success.status() << " " + << success.value(); - // Clean up. - sapi::StatusOr result = api.c_jsonnet_realloc(&vm_pointer, &output_pointer, 0); - CHECK(result.ok()) << "JsonnetVm realloc failed " << result.status(); + // Clean up. + sapi::StatusOr result = + api.c_jsonnet_realloc(&vm_pointer, &output_pointer, 0); + CHECK(result.ok()) << "JsonnetVm realloc failed " << result.status(); - status = api.c_jsonnet_destroy(&vm_pointer); - CHECK(status.ok()) << "JsonnetVm destroy failed " << status; + status = api.c_jsonnet_destroy(&vm_pointer); + CHECK(status.ok()) << "JsonnetVm destroy failed " << status; - status = api.c_free_input(&input_pointer); - CHECK(status.ok()) << "Input freeing failed " << status; + status = api.c_free_input(&input_pointer); + CHECK(status.ok()) << "Input freeing failed " << status; - return EXIT_SUCCESS; + return EXIT_SUCCESS; } \ No newline at end of file diff --git a/oss-internship-2020/jsonnet/jsonnet_helper.cc b/oss-internship-2020/jsonnet/jsonnet_helper.cc index 2b0270e..86af04a 100644 --- a/oss-internship-2020/jsonnet/jsonnet_helper.cc +++ b/oss-internship-2020/jsonnet/jsonnet_helper.cc @@ -13,39 +13,39 @@ // limitations under the License. #include "jsonnet_helper.h" + #include struct JsonnetVm* c_jsonnet_make(void) { - return jsonnet_make(); + return jsonnet_make(); } -void c_jsonnet_destroy(struct JsonnetVm* vm) { - return jsonnet_destroy(vm); -} +void c_jsonnet_destroy(struct JsonnetVm* vm) { return jsonnet_destroy(vm); } -char* c_jsonnet_evaluate_snippet(struct JsonnetVm* vm, const char* filename, char* snippet, int* error) { - return jsonnet_evaluate_snippet(vm, filename, snippet, error); +char* c_jsonnet_evaluate_snippet(struct JsonnetVm* vm, const char* filename, + char* snippet, int* error) { + return jsonnet_evaluate_snippet(vm, filename, snippet, error); } char* c_read_input(bool filename_is_code, const char* filename) { - std::string s_filename(filename); - std::string s_input; - bool check = read_input(filename_is_code, &s_filename, &s_input); - char* c_input = strdup(s_input.c_str()); - if (check) return c_input; - return nullptr; + std::string s_filename(filename); + std::string s_input; + bool check = read_input(filename_is_code, &s_filename, &s_input); + char* c_input = strdup(s_input.c_str()); + if (check) return c_input; + return nullptr; } void c_free_input(char* input) { - free(input); - return; + free(input); + return; } bool c_write_output_file(const char* output, const char* output_file) { - std::string s_output_file(output_file); - return write_output_file(output, s_output_file); + std::string s_output_file(output_file); + return write_output_file(output, s_output_file); } char* c_jsonnet_realloc(JsonnetVm* vm, char* str, size_t sz) { - return jsonnet_realloc(vm, str, sz); + return jsonnet_realloc(vm, str, sz); } \ No newline at end of file diff --git a/oss-internship-2020/jsonnet/jsonnet_helper.h b/oss-internship-2020/jsonnet/jsonnet_helper.h index a586ac0..9888774 100644 --- a/oss-internship-2020/jsonnet/jsonnet_helper.h +++ b/oss-internship-2020/jsonnet/jsonnet_helper.h @@ -22,12 +22,15 @@ extern "C" struct JsonnetVm* c_jsonnet_make(void); extern "C" void c_jsonnet_destroy(struct JsonnetVm* vm); -extern "C" char* c_jsonnet_evaluate_snippet(struct JsonnetVm* vm, const char* filename, char* snippet, int* error); +extern "C" char* c_jsonnet_evaluate_snippet(struct JsonnetVm* vm, + const char* filename, char* snippet, + int* error); extern "C" char* c_read_input(bool filename_is_code, const char* filename); extern "C" void c_free_input(char* input); -extern "C" bool c_write_output_file(const char* output, const char* output_file); +extern "C" bool c_write_output_file(const char* output, + const char* output_file); extern "C" char* c_jsonnet_realloc(JsonnetVm* vm, char* str, size_t sz); \ No newline at end of file From dfc9d11f352ba61169d9d161838d712722e5baa6 Mon Sep 17 00:00:00 2001 From: Katarzyna Miernikiewicz Date: Tue, 8 Sep 2020 15:03:22 +0000 Subject: [PATCH 04/28] enabling usage of files in the same directory as input file --- .../jsonnet/examples/jsonnet_example.cc | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc index 04e116c..a8c16d9 100644 --- a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc +++ b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc @@ -21,6 +21,8 @@ #include "jsonnet_sapi.sapi.h" #include "sandboxed_api/util/flag.h" +ABSL_DECLARE_FLAG(string, sandbox2_danger_danger_permit_all_and_log); + class JsonnetSapiSandbox : public JsonnetSandbox { public: explicit JsonnetSapiSandbox(std::string in_file, std::string out_file) @@ -40,8 +42,9 @@ class JsonnetSapiSandbox : public JsonnetSandbox { __NR_futex, __NR_close, }) - .AddFile(in_file_) .AddDirectoryAt(dirname(&out_file_[0]), "/output", /*is_ro=*/false) + .AddDirectoryAt(dirname(&in_file_[0]), "/input", true) + // .AddFileAt(in_file_, std::string("/input/"), true) .BuildOrDie(); } @@ -78,7 +81,9 @@ int main(int argc, char *argv[]) { << jsonnet_vm.status(); // Read input file. - sapi::v::ConstCStr in_file_var(in_file.c_str()); + std::string in_file_in_sandboxee(std::string("/input/") + + basename(&in_file[0])); + sapi::v::ConstCStr in_file_var(in_file_in_sandboxee.c_str()); sapi::StatusOr input = api.c_read_input(false, in_file_var.PtrBefore()); CHECK(input.ok()) << "Reading input file failed " << input.status(); @@ -89,8 +94,10 @@ int main(int argc, char *argv[]) { sapi::StatusOr output = api.c_jsonnet_evaluate_snippet( &vm_pointer, in_file_var.PtrBefore(), &input_pointer, error.PtrAfter()); CHECK(output.ok() && !error.GetValue()) - << "Jsonnet code evaluation failed" << output.status() << " " - << error.GetValue(); + << "Jsonnet code evaluation failed: " << output.status() << " " + << error.GetValue() << "\n" + << "Make sure all files used by your jsonnet file are in the same " + "directory as your file"; // Write data to file. std::string out_file_in_sandboxee(std::string("/output/") + @@ -106,13 +113,13 @@ int main(int argc, char *argv[]) { // Clean up. sapi::StatusOr result = api.c_jsonnet_realloc(&vm_pointer, &output_pointer, 0); - CHECK(result.ok()) << "JsonnetVm realloc failed " << result.status(); + CHECK(result.ok()) << "JsonnetVm realloc failed: " << result.status(); status = api.c_jsonnet_destroy(&vm_pointer); - CHECK(status.ok()) << "JsonnetVm destroy failed " << status; + CHECK(status.ok()) << "JsonnetVm destroy failed: " << status; status = api.c_free_input(&input_pointer); - CHECK(status.ok()) << "Input freeing failed " << status; + CHECK(status.ok()) << "Input freeing failed: " << status; return EXIT_SUCCESS; } \ No newline at end of file From f7eff228f009d9a7172160f56aa1fb054713e1da Mon Sep 17 00:00:00 2001 From: Katarzyna Miernikiewicz Date: Tue, 8 Sep 2020 15:28:21 +0000 Subject: [PATCH 05/28] minor fix --- oss-internship-2020/jsonnet/examples/jsonnet_example.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc index a8c16d9..ba65583 100644 --- a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc +++ b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc @@ -44,7 +44,6 @@ class JsonnetSapiSandbox : public JsonnetSandbox { }) .AddDirectoryAt(dirname(&out_file_[0]), "/output", /*is_ro=*/false) .AddDirectoryAt(dirname(&in_file_[0]), "/input", true) - // .AddFileAt(in_file_, std::string("/input/"), true) .BuildOrDie(); } From 1b307adf5d55284f6776067d1f627d371c974fe3 Mon Sep 17 00:00:00 2001 From: Katarzyna Miernikiewicz Date: Wed, 9 Sep 2020 09:56:42 +0000 Subject: [PATCH 06/28] added README --- oss-internship-2020/jsonnet/README.md | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 oss-internship-2020/jsonnet/README.md diff --git a/oss-internship-2020/jsonnet/README.md b/oss-internship-2020/jsonnet/README.md new file mode 100644 index 0000000..e0f2e9a --- /dev/null +++ b/oss-internship-2020/jsonnet/README.md @@ -0,0 +1,29 @@ +# Jsonnet Sandboxed API + +This library provides sandboxed version of the [Jsonnet](https://github.com/google/jsonnet) library. + +## Examples + +For now the only example command-line tool `jsonnet_sandboxed` enables the user to evaluate jsonnet code held in one file and writing to one output file. The tool is based on what can be found [here](https://github.com/google/jsonnet/blob/master/cmd/jsonnet.cpp) -- . + +## Build + +To build this example, after cloning the whole Sandbox API project, you also need to run + +``` +git submodule update --init --recursive +``` +anywhere in the project tree in order to clone the `jsonnet` submodule. +Then in the `sandboxed-api/oss-internship-2020/jsonnet` run +``` +mkdir build && cd build +cmake -G Ninja +ninja +``` +To run `jsonnet_sandboxed`: +``` +cd examples +./jsonnet_sandboxed absolute/path/to/the/input_file.jsonnet \ + absolute/path/to/the/output_file +``` +For now it supports evaluating one input file (possibly relying on multiple other files, e.x. by jsonnet `import` command; the files must be held in the same directory as input file) into one output file. Example jsonnet codes to evaluate can be found [here](https://github.com/google/jsonnet/tree/master/examples). From 76506576623bcf2cc5a13307e4d29adabbefe6c7 Mon Sep 17 00:00:00 2001 From: Katarzyna Miernikiewicz Date: Thu, 10 Sep 2020 10:19:06 +0000 Subject: [PATCH 07/28] multiple files example added --- oss-internship-2020/jsonnet/CMakeLists.txt | 17 +- oss-internship-2020/jsonnet/README.md | 11 +- .../jsonnet/examples/CMakeLists.txt | 38 + .../jsonnet/examples/jsonnet_example.cc | 15 +- .../jsonnet_multiple_files_example.cc | 119 ++++ .../examples/multiple_files_example.jsonnet | 12 + oss-internship-2020/jsonnet/jsonnet.patch | 668 ++++++++++++++++++ oss-internship-2020/jsonnet/jsonnet_helper.cc | 9 + oss-internship-2020/jsonnet/jsonnet_helper.h | 8 +- 9 files changed, 873 insertions(+), 24 deletions(-) create mode 100644 oss-internship-2020/jsonnet/examples/jsonnet_multiple_files_example.cc create mode 100644 oss-internship-2020/jsonnet/examples/multiple_files_example.jsonnet create mode 100644 oss-internship-2020/jsonnet/jsonnet.patch diff --git a/oss-internship-2020/jsonnet/CMakeLists.txt b/oss-internship-2020/jsonnet/CMakeLists.txt index 9054beb..f858d60 100644 --- a/oss-internship-2020/jsonnet/CMakeLists.txt +++ b/oss-internship-2020/jsonnet/CMakeLists.txt @@ -21,16 +21,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) add_subdirectory(jsonnet) -add_library(jsonnet_helper STATIC - jsonnet_helper.cc - jsonnet_helper.h - ${CMAKE_CURRENT_SOURCE_DIR}/jsonnet/cmd/utils.h - ${CMAKE_CURRENT_SOURCE_DIR}/jsonnet/cmd/utils.cpp -) - -target_link_libraries(jsonnet_helper - libjsonnet_for_binaries -) +add_subdirectory(examples) set(SAPI_ROOT "../.." CACHE PATH "Path to the Sandboxed API source tree") set(SAPI_ENABLE_EXAMPLES OFF CACHE BOOL "") @@ -48,6 +39,8 @@ add_sapi_library(jsonnet_sapi c_free_input c_write_output_file c_jsonnet_realloc + c_jsonnet_evaluate_snippet_multi + c_write_multi_output_files INPUTS jsonnet_helper.h LIBRARY jsonnet_helper LIBRARY_NAME Jsonnet @@ -58,6 +51,4 @@ target_include_directories(jsonnet_sapi INTERFACE "${PROJECT_BINARY_DIR}" ) -target_link_libraries(jsonnet_sapi PUBLIC jsonnet_helper) - -add_subdirectory(examples) +target_link_libraries(jsonnet_sapi PUBLIC jsonnet_helper) \ No newline at end of file diff --git a/oss-internship-2020/jsonnet/README.md b/oss-internship-2020/jsonnet/README.md index e0f2e9a..b71279f 100644 --- a/oss-internship-2020/jsonnet/README.md +++ b/oss-internship-2020/jsonnet/README.md @@ -4,7 +4,8 @@ This library provides sandboxed version of the [Jsonnet](https://github.com/goog ## Examples -For now the only example command-line tool `jsonnet_sandboxed` enables the user to evaluate jsonnet code held in one file and writing to one output file. The tool is based on what can be found [here](https://github.com/google/jsonnet/blob/master/cmd/jsonnet.cpp) -- . +The `examples/` directory contains code to produce two command-line tools -- `jsonnet_sandboxed` and `jsonnet_multiple_files_sandboxed`. The first one enables the user to evaluate jsonnet code held in one file and writing to one output file. The other one is for evaluating one jsonnet file into multiple output files. +Both tools are based on what can be found [here](https://github.com/google/jsonnet/blob/master/cmd/jsonnet.cpp). ## Build @@ -26,4 +27,10 @@ cd examples ./jsonnet_sandboxed absolute/path/to/the/input_file.jsonnet \ absolute/path/to/the/output_file ``` -For now it supports evaluating one input file (possibly relying on multiple other files, e.x. by jsonnet `import` command; the files must be held in the same directory as input file) into one output file. Example jsonnet codes to evaluate can be found [here](https://github.com/google/jsonnet/tree/master/examples). +To run `jsonnet_mutiple_files_sandboxed`: +``` +cd examples +./jsonnet_mutiple_files_sandboxed absolute/path/to/the/input_file.jsonnet \ + absolute/path/to/the/output_directory +``` +Both tools support evaluating one input file (possibly relying on multiple other files, e.x. by jsonnet `import` command; the files must be held in the same directory as input file) into one or more output files. Example jsonnet codes to evaluate in a one-in-one-out manner can be found [here](https://github.com/google/jsonnet/tree/master/examples). Example code producing multiple output files can be found in the `examples` directory, in a file called `multiple_files_example.jsonnet`. \ No newline at end of file diff --git a/oss-internship-2020/jsonnet/examples/CMakeLists.txt b/oss-internship-2020/jsonnet/examples/CMakeLists.txt index be008f5..b67e534 100644 --- a/oss-internship-2020/jsonnet/examples/CMakeLists.txt +++ b/oss-internship-2020/jsonnet/examples/CMakeLists.txt @@ -12,13 +12,51 @@ # See the License for the specific language governing permissions and # limitations under the License. +file(MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/gen_files") +file(COPY "${PROJECT_SOURCE_DIR}/jsonnet/cmd/jsonnet.cpp" DESTINATION "${PROJECT_BINARY_DIR}/gen_files/") +file(COPY "${PROJECT_SOURCE_DIR}/jsonnet.patch" DESTINATION "${PROJECT_BINARY_DIR}/gen_files/") + +add_custom_command( + OUTPUT ${PROJECT_BINARY_DIR}/gen_files/write_helper.cc + COMMAND cd ${PROJECT_BINARY_DIR}/gen_files && patch < ${PROJECT_SOURCE_DIR}/jsonnet.patch > /dev/null + COMMAND mv ${PROJECT_BINARY_DIR}/gen_files/jsonnet.cpp ${PROJECT_BINARY_DIR}/gen_files/write_helper.cc +) + +include_directories( + ${PROJECT_SOURCE_DIR} + ${PROJECT_BINARY_DIR}/gen_files +) + +add_library(jsonnet_helper STATIC + ${PROJECT_SOURCE_DIR}/jsonnet_helper.cc + ${PROJECT_SOURCE_DIR}/jsonnet_helper.h + ${PROJECT_SOURCE_DIR}/jsonnet/cmd/utils.h + ${PROJECT_SOURCE_DIR}/jsonnet/cmd/utils.cpp + ${PROJECT_BINARY_DIR}/gen_files/write_helper.cc +) + +target_link_libraries(jsonnet_helper + libjsonnet_for_binaries +) + add_executable(jsonnet_sandboxed jsonnet_example.cc ) +add_executable(jsonnet_mutiple_files_sandboxed + jsonnet_multiple_files_example.cc +) + target_link_libraries(jsonnet_sandboxed PRIVATE libjsonnet jsonnet_helper jsonnet_sapi sapi::sapi ) + +target_link_libraries(jsonnet_mutiple_files_sandboxed PRIVATE + libjsonnet + jsonnet_helper + jsonnet_sapi + sapi::sapi +) diff --git a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc index ba65583..3747448 100644 --- a/oss-internship-2020/jsonnet/examples/jsonnet_example.cc +++ b/oss-internship-2020/jsonnet/examples/jsonnet_example.cc @@ -19,9 +19,6 @@ #include #include "jsonnet_sapi.sapi.h" -#include "sandboxed_api/util/flag.h" - -ABSL_DECLARE_FLAG(string, sandbox2_danger_danger_permit_all_and_log); class JsonnetSapiSandbox : public JsonnetSandbox { public: @@ -56,9 +53,9 @@ int main(int argc, char *argv[]) { google::InitGoogleLogging(argv[0]); gflags::ParseCommandLineFlags(&argc, &argv, true); - if (argc != 3) { - std::cerr << "Usage: " << basename(argv[0]) - << " absolute/path/to/INPUT.jsonnet" + if (!(argc == 3)) { + std::cerr << "Usage:\n" + << basename(argv[0]) << " absolute/path/to/INPUT.jsonnet" << " absolute/path/to/OUTPUT\n"; return EXIT_FAILURE; } @@ -103,8 +100,10 @@ int main(int argc, char *argv[]) { basename(&out_file[0])); sapi::v::ConstCStr out_file_var(out_file_in_sandboxee.c_str()); sapi::v::RemotePtr output_pointer(output.value()); - sapi::StatusOr success = - api.c_write_output_file(&output_pointer, out_file_var.PtrBefore()); + sapi::StatusOr success; + + success = api.c_write_output_file(&output_pointer, out_file_var.PtrBefore()); + CHECK(success.ok() && success.value()) << "Writing to output file failed " << success.status() << " " << success.value(); diff --git a/oss-internship-2020/jsonnet/examples/jsonnet_multiple_files_example.cc b/oss-internship-2020/jsonnet/examples/jsonnet_multiple_files_example.cc new file mode 100644 index 0000000..8f8878c --- /dev/null +++ b/oss-internship-2020/jsonnet/examples/jsonnet_multiple_files_example.cc @@ -0,0 +1,119 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include +#include + +#include "jsonnet_sapi.sapi.h" +#include "sandboxed_api/sandbox2/util/path.h" + +class JsonnetSapiSandbox : public JsonnetSandbox { + public: + explicit JsonnetSapiSandbox(std::string in_file, std::string out_file) + : in_file_(std::move(in_file)), out_file_(std::move(out_file)) {} + + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder *) override { + return sandbox2::PolicyBuilder() + .AllowStaticStartup() + .AllowOpen() + .AllowRead() + .AllowWrite() + .AllowStat() + .AllowSystemMalloc() + .AllowExit() + .AllowSyscalls({ + __NR_futex, + __NR_close, + }) + .AddDirectoryAt(sandbox2::file::CleanPath(&out_file_[0]), "/output", + /*is_ro=*/false) + .AddDirectoryAt(dirname(&in_file_[0]), "/input", true) + .BuildOrDie(); + } + + private: + std::string in_file_; + std::string out_file_; +}; + +int main(int argc, char *argv[]) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + + if (!(argc == 3)) { + std::cerr << "Usage:\n" + << basename(argv[0]) << " absolute/path/to/INPUT.jsonnet" + << " absolute/path/to/OUTPUT_DIRECTORY\n"; + return EXIT_FAILURE; + } + + std::string in_file(argv[1]); + std::string out_directory(argv[2]); + + // Initialize sandbox. + JsonnetSapiSandbox sandbox(in_file, out_directory); + absl::Status status = sandbox.Init(); + CHECK(status.ok()) << "Sandbox initialization failed " << status; + + JsonnetApi api(&sandbox); + + // Initialize library's main structure. + sapi::StatusOr jsonnet_vm = api.c_jsonnet_make(); + sapi::v::RemotePtr vm_pointer(jsonnet_vm.value()); + CHECK(jsonnet_vm.ok()) << "JsonnetVm initialization failed: " + << jsonnet_vm.status(); + + // Read input file. + std::string in_file_in_sandboxee(std::string("/input/") + + basename(&in_file[0])); + sapi::v::ConstCStr in_file_var(in_file_in_sandboxee.c_str()); + sapi::StatusOr input = + api.c_read_input(false, in_file_var.PtrBefore()); + CHECK(input.ok()) << "Reading input file failed " << input.status(); + + // Process jsonnet data. + sapi::v::RemotePtr input_pointer(input.value()); + sapi::v::Int error; + sapi::StatusOr output = api.c_jsonnet_evaluate_snippet_multi( + &vm_pointer, in_file_var.PtrBefore(), &input_pointer, error.PtrAfter()); + CHECK(output.ok() && !error.GetValue()) + << "Jsonnet code evaluation failed: " << output.status() << " " + << error.GetValue() << "\n" + << "Make sure all files used by your jsonnet file are in the same " + "directory as your file"; + + // Write data to file. + std::string out_file_in_sandboxee(std::string("/output/")); + sapi::v::ConstCStr out_file_var(out_file_in_sandboxee.c_str()); + sapi::v::RemotePtr output_pointer(output.value()); + sapi::StatusOr success = api.c_write_multi_output_files( + &vm_pointer, &output_pointer, out_file_var.PtrBefore()); + + CHECK(success.ok() && success.value()) + << "Writing to output file failed " << success.status() << " " + << success.value(); + + // Clean up. + status = api.c_jsonnet_destroy(&vm_pointer); + CHECK(status.ok()) << "JsonnetVm destroy failed: " << status; + + status = api.c_free_input(&input_pointer); + CHECK(status.ok()) << "Input freeing failed: " << status; + + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/oss-internship-2020/jsonnet/examples/multiple_files_example.jsonnet b/oss-internship-2020/jsonnet/examples/multiple_files_example.jsonnet new file mode 100644 index 0000000..974c8df --- /dev/null +++ b/oss-internship-2020/jsonnet/examples/multiple_files_example.jsonnet @@ -0,0 +1,12 @@ +// This is a jsonnet code which evaluates to mutliple output files. +{ + "first_file.json": { + name: 'This is the first file created by the multiple-files example code.', + caption: 'The other one\'s name is -> ' + $["second_file.json"].name, + }, + "second_file.json": { + name: 'And that is the other one.', + caption: 'If it was the first one, variable name would hold what\'s in variable.', + first_name: $["first_file.json"].name, + }, +} \ No newline at end of file diff --git a/oss-internship-2020/jsonnet/jsonnet.patch b/oss-internship-2020/jsonnet/jsonnet.patch new file mode 100644 index 0000000..9fc7939 --- /dev/null +++ b/oss-internship-2020/jsonnet/jsonnet.patch @@ -0,0 +1,668 @@ +--- jsonnet.cpp 2020-09-09 12:15:33.687539042 +0000 ++++ write_helper.cpp 2020-09-09 14:45:55.176665636 +0000 +@@ -14,559 +14,126 @@ + limitations under the License. + */ + +-#include +-#include +-#include ++// We need two functions defined in jsonnet.cpp file, used for writing output ++// (multiple files and yaml streams) -- with minor changes (e.x. return type). + +-#include + #include + #include +-#include + #include +-#include +-#include + #include + +-#include "utils.h" ++#include "jsonnet_helper.h" + +-extern "C" { +-#include +-} +- +-#ifdef _WIN32 +-const char PATH_SEP = ';'; +-#else +-const char PATH_SEP = ':'; +-#endif +- +-void version(std::ostream &o) +-{ +- o << "Jsonnet commandline interpreter " << jsonnet_version() << std::endl; +-} +- +-void usage(std::ostream &o) +-{ +- version(o); +- o << "\n"; +- o << "jsonnet {