YAML stream example added

oss-internship-2020/jsonnet/CMakeLists.txt

@@ -41,6 +41,8 @@ add_sapi_library(jsonnet_sapi
             c_jsonnet_realloc
             c_jsonnet_evaluate_snippet_multi
             c_write_multi_output_files
+            c_write_output_stream
+            c_jsonnet_evaluate_snippet_stream
   INPUTS jsonnet_helper.h
   LIBRARY jsonnet_helper
   LIBRARY_NAME Jsonnet

oss-internship-2020/jsonnet/README.md

@@ -4,8 +4,8 @@ This library provides sandboxed version of the [Jsonnet](https://github.com/goog
 
 ## Examples
 
-The `examples/` directory contains code to produce two command-line tools -- `jsonnet_sandboxed` and `jsonnet_multiple_files_sandboxed`. The first one enables the user to evaluate jsonnet code held in one file and writing to one output file. The other one is for evaluating one jsonnet file into multiple output files.
+The `examples/` directory contains code to produce three command-line tools -- `jsonnet_sandboxed`, `jsonnet_yaml_stream_sandboxed` and `jsonnet_multiple_files_sandboxed`. The first one enables the user to evaluate jsonnet code held in one file and writing to one output file. The second evaluates one jsonnet file into one file, which can be interepreted as YAML stream. The third one is for evaluating one jsonnet file into multiple output files.
-Both tools are based on what can be found [here](https://github.com/google/jsonnet/blob/master/cmd/jsonnet.cpp).
+All three tools are based on what can be found [here](https://github.com/google/jsonnet/blob/master/cmd/jsonnet.cpp).
 
 ## Build
 
@@ -21,7 +21,7 @@ mkdir build && cd build
 cmake -G Ninja
 ninja
 ```
-To run `jsonnet_sandboxed`:
+To run `jsonnet_sandboxed` (or `jsonnet_yaml_stream_sandboxed` in a similar way):
 ```
 cd examples
 ./jsonnet_sandboxed absolute/path/to/the/input_file.jsonnet \ 
@@ -33,4 +33,4 @@ cd examples
 ./jsonnet_mutiple_files_sandboxed absolute/path/to/the/input_file.jsonnet \ 
     absolute/path/to/the/output_directory
 ```
-Both tools support evaluating one input file (possibly relying on multiple other files, e.x. by jsonnet `import` command; the files must be held in the same directory as input file) into one or more output files. Example jsonnet codes to evaluate in a one-in-one-out manner can be found [here](https://github.com/google/jsonnet/tree/master/examples). Example code producing multiple output files can be found in the `examples` directory, in a file called `multiple_files_example.jsonnet`.
+All three tools support evaluating one input file (possibly relying on multiple other files, e.x. by jsonnet `import` command; the files must be held in the same directory as input file) into one or more output files. Example jsonnet codes to evaluate in a one-in-one-out manner can be found [here](https://github.com/google/jsonnet/tree/master/examples). Example code producing multiple output files or YAML stream files can be found in the `examples/jsonnet_codes` directory, in files called `multiple_files_example.jsonnet` and `yaml_stream_example.jsonnet`, respectively.

oss-internship-2020/jsonnet/examples/CMakeLists.txt

@@ -39,24 +39,14 @@ target_link_libraries(jsonnet_helper
   libjsonnet_for_binaries
 )
 
-add_executable(jsonnet_sandboxed
+foreach(exe base multiple_files yaml_stream)
-    jsonnet_example.cc
+  add_executable(jsonnet_${exe}_sandboxed
-)
+    jsonnet_${exe}_example.cc
-
+  )
-add_executable(jsonnet_mutiple_files_sandboxed
+  target_link_libraries(jsonnet_${exe}_sandboxed PRIVATE
-    jsonnet_multiple_files_example.cc
-)
-
-target_link_libraries(jsonnet_sandboxed PRIVATE
     libjsonnet
     jsonnet_helper
     jsonnet_sapi
     sapi::sapi
-)
+  )
-
+endforeach()
-target_link_libraries(jsonnet_mutiple_files_sandboxed PRIVATE
-  libjsonnet
-  jsonnet_helper
-  jsonnet_sapi
-  sapi::sapi
-)

oss-internship-2020/jsonnet/examples/jsonnet_codes/yaml_stream_example.jsonnet

@@ -2,10 +2,10 @@
 local
   first_object = {
     name: 'First object\'s name.',
-    age: 'Just created!,
+    age: 'Just created!',
   },
   second_object = {
-    name: `Hi, my name is <second_object>.`,
+    name: 'Hi, my name is <second_object>.',
     sibling: first_object.name
   };
 

oss-internship-2020/jsonnet/examples/jsonnet_multiple_files_example.cc

@@ -23,8 +23,8 @@
 
 class JsonnetSapiSandbox : public JsonnetSandbox {
  public:
-  explicit JsonnetSapiSandbox(std::string in_file, std::string out_file)
+  explicit JsonnetSapiSandbox(std::string in_file, std::string out_directory)
-      : in_file_(std::move(in_file)), out_file_(std::move(out_file)) {}
+      : in_file_(std::move(in_file)), out_directory_(std::move(out_directory)) {}
 
   std::unique_ptr<sandbox2::Policy> ModifyPolicy(
       sandbox2::PolicyBuilder *) override {
@@ -40,7 +40,7 @@ class JsonnetSapiSandbox : public JsonnetSandbox {
             __NR_futex,
             __NR_close,
         })
-        .AddDirectoryAt(sandbox2::file::CleanPath(&out_file_[0]), "/output",
+        .AddDirectoryAt(sandbox2::file::CleanPath(&out_directory_[0]), "/output",
                         /*is_ro=*/false)
         .AddDirectoryAt(dirname(&in_file_[0]), "/input", true)
         .BuildOrDie();
@@ -48,7 +48,7 @@ class JsonnetSapiSandbox : public JsonnetSandbox {
 
  private:
   std::string in_file_;
-  std::string out_file_;
+  std::string out_directory_;
 };
 
 int main(int argc, char *argv[]) {

oss-internship-2020/jsonnet/examples/jsonnet_yaml_stream_example.cc

@@ -0,0 +1,119 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <libgen.h>
+#include <syscall.h>
+
+#include <cstdlib>
+#include <iostream>
+
+#include "jsonnet_sapi.sapi.h"
+
+class JsonnetSapiSandbox : public JsonnetSandbox {
+ public:
+  explicit JsonnetSapiSandbox(std::string in_file, std::string out_file)
+      : in_file_(std::move(in_file)), out_file_(std::move(out_file)) {}
+
+  std::unique_ptr<sandbox2::Policy> ModifyPolicy(
+      sandbox2::PolicyBuilder *) override {
+    return sandbox2::PolicyBuilder()
+        .AllowStaticStartup()
+        .AllowOpen()
+        .AllowRead()
+        .AllowWrite()
+        .AllowStat()
+        .AllowSystemMalloc()
+        .AllowExit()
+        .AllowSyscalls({
+            __NR_futex,
+            __NR_close,
+        })
+        .AddDirectoryAt(dirname(&out_file_[0]), "/output", /*is_ro=*/false)
+        .AddDirectoryAt(dirname(&in_file_[0]), "/input", true)
+        .BuildOrDie();
+  }
+
+ private:
+  std::string in_file_;
+  std::string out_file_;
+};
+
+int main(int argc, char *argv[]) {
+  google::InitGoogleLogging(argv[0]);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  if (!(argc == 3)) {
+    std::cerr << "Usage:\n"
+              << basename(argv[0]) << " absolute/path/to/INPUT.jsonnet"
+              << " absolute/path/to/OUTPUT\n";
+    return EXIT_FAILURE;
+  }
+
+  std::string in_file(argv[1]);
+  std::string out_file(argv[2]);
+
+  // Initialize sandbox.
+  JsonnetSapiSandbox sandbox(in_file, out_file);
+  absl::Status status = sandbox.Init();
+  CHECK(status.ok()) << "Sandbox initialization failed " << status;
+
+  JsonnetApi api(&sandbox);
+
+  // Initialize library's main structure.
+  sapi::StatusOr<JsonnetVm *> jsonnet_vm = api.c_jsonnet_make();
+  sapi::v::RemotePtr vm_pointer(jsonnet_vm.value());
+  CHECK(jsonnet_vm.ok()) << "JsonnetVm initialization failed: "
+                         << jsonnet_vm.status();
+
+  // Read input file.
+  std::string in_file_in_sandboxee(std::string("/input/") +
+                                   basename(&in_file[0]));
+  sapi::v::ConstCStr in_file_var(in_file_in_sandboxee.c_str());
+  sapi::StatusOr<char *> input =
+      api.c_read_input(false, in_file_var.PtrBefore());
+  CHECK(input.ok()) << "Reading input file failed " << input.status();
+
+  // Process jsonnet data.
+  sapi::v::RemotePtr input_pointer(input.value());
+  sapi::v::Int error;
+  sapi::StatusOr<char *> output = api.c_jsonnet_evaluate_snippet_stream(
+      &vm_pointer, in_file_var.PtrBefore(), &input_pointer, error.PtrAfter());
+  CHECK(output.ok() && !error.GetValue())
+      << "Jsonnet code evaluation failed: " << output.status() << " "
+      << error.GetValue() << "\n"
+      << "Make sure all files used by your jsonnet file are in the same "
+         "directory as your file";
+
+  // Write data to file.
+  std::string out_file_in_sandboxee(std::string("/output/") +
+                                    basename(&out_file[0]));
+  sapi::v::ConstCStr out_file_var(out_file_in_sandboxee.c_str());
+  sapi::v::RemotePtr output_pointer(output.value());
+  sapi::StatusOr<bool> success;
+
+  success = api.c_write_output_stream(&vm_pointer, &output_pointer,
+                                      out_file_var.PtrBefore());
+  CHECK(success.ok() && success.value())
+      << "Writing to output file failed " << success.status() << " "
+      << success.value();
+
+  // Clean up.
+  status = api.c_jsonnet_destroy(&vm_pointer);
+  CHECK(status.ok()) << "JsonnetVm destroy failed: " << status;
+
+  status = api.c_free_input(&input_pointer);
+  CHECK(status.ok()) << "Input freeing failed: " << status;
+
+  return EXIT_SUCCESS;
+}

oss-internship-2020/jsonnet/jsonnet.patch

@@ -1,5 +1,5 @@
 --- jsonnet.cpp	2020-09-09 12:15:33.687539042 +0000
-+++ write_helper.cpp	2020-09-09 14:45:55.176665636 +0000
++++ write_helper.cpp	2020-09-10 11:21:44.273825853 +0000
 @@ -14,559 +14,126 @@
  limitations under the License.
  */
@@ -143,17 +143,7 @@
 -        if (val_cstr == nullptr) {
 -            std::cerr << "ERROR: environment variable " << var << " was undefined." << std::endl;
 -            return false;
-+      std::ifstream exists(filename.c_str());
+-        }
-+      if (exists.good()) {
-+        std::string existing_content;
-+        existing_content.assign(std::istreambuf_iterator<char>(exists),
-+                                std::istreambuf_iterator<char>());
-+        if (existing_content == new_content) {
-+          // Do not bump the timestamp on the file if its content is
-+          // the same. This may trigger other tools (e.g. make) to do
-+          // unnecessary work.
-+          continue;
-         }
 -        val = val_cstr;
 -    } else {
 -        var = var_val.substr(0, eq_pos);
@@ -169,8 +159,7 @@
 -        std::cerr << "ERROR: argument not in form <var>=<file> \"" << var_file << "\"."
 -                  << std::endl;
 -        return false;
-+      }
+-    }
-     }
 -    var = var_file.substr(0, eq_pos);
 -    const std::string path = var_file.substr(eq_pos + 1, std::string::npos);
 -
@@ -361,8 +350,19 @@
 -            return ARG_FAILURE;
 -        } else {
 -            remaining_args.push_back(args[i]);
--        }
++      std::ifstream exists(filename.c_str());
--    }
++      if (exists.good()) {
++        std::string existing_content;
++        existing_content.assign(std::istreambuf_iterator<char>(exists),
++                                std::istreambuf_iterator<char>());
++        if (existing_content == new_content) {
++          // Do not bump the timestamp on the file if its content is
++          // the same. This may trigger other tools (e.g. make) to do
++          // unnecessary work.
++          continue;
+         }
++      }
+     }
 -
 -    const char *want = config->filenameIsCode ? "code" : "filename";
 -    if (remaining_args.size() == 0) {
@@ -416,8 +416,21 @@
 -            return false;
 -        }
 -        o = &f;
--    }
++    f.open(filename.c_str());
--
++    if (!f.good()) {
++      std::string msg = "Opening output file: " + filename;
++      perror(msg.c_str());
++      return false;
++    }
++    f << new_content;
++    f.close();
++    if (!f.good()) {
++      std::string msg = "Writing to output file: " + filename;
++      perror(msg.c_str());
++      return false;
+     }
++  }
+ 
 -    for (const auto &pair : r) {
 -        const std::string &new_content = pair.second;
 -        const std::string &filename = output_dir + pair.first;
@@ -451,7 +464,8 @@
 -            return false;
 -        }
 -    }
--
++  std::cout.flush();
+ 
 -    if (output_file.empty()) {
 -        std::cout.flush();
 -    } else {
@@ -463,23 +477,6 @@
 -        }
 -    }
 -    return true;
-+    f.open(filename.c_str());
-+    if (!f.good()) {
-+      std::string msg = "Opening output file: " + filename;
-+      perror(msg.c_str());
-+      return false;
-+    }
-+    f << new_content;
-+    f.close();
-+    if (!f.good()) {
-+      std::string msg = "Writing to output file: " + filename;
-+      perror(msg.c_str());
-+      return false;
-+    }
-+  }
-+
-+  std::cout.flush();
-+  
 +  return true;
  }
  
@@ -499,38 +496,7 @@
 -            return false;
 -        }
 -        o = &f;
--    }
++bool write_output_stream(JsonnetVm *vm, char *output,
--
--    // If YAML stream output is used, then iterate over each string from
--    // the sequence of strings returned by jsonnet_evaluate_snippet_stream,
--    // and add the --- and ... as defined by the YAML spec.
--    std::vector<std::string> r;
--    for (const char *c = output; *c != '\0';) {
--        const char *json = c;
--        while (*c != '\0')
--            ++c;
--        ++c;
--        r.emplace_back(json);
--    }
--    jsonnet_realloc(vm, output, 0);
--    for (const auto &str : r) {
--        (*o) << "---\n";
--        (*o) << str;
--    }
--    if (r.size() > 0)
--        (*o) << "...\n";
--    o->flush();
--
--    if (output_file.empty()) {
--        std::cout.flush();
--    } else {
--        f.close();
--        if (!f.good()) {
--            std::string msg = "Writing to output file: " + output_file;
--            perror(msg.c_str());
--            return false;
--        }
-+static bool write_output_stream(JsonnetVm *vm, char *output,
 +                         const std::string &output_file) {
 +  std::ostream *o;
 +  std::ofstream f;
@@ -576,6 +542,37 @@
      }
 +  }
  
+-    // If YAML stream output is used, then iterate over each string from
+-    // the sequence of strings returned by jsonnet_evaluate_snippet_stream,
+-    // and add the --- and ... as defined by the YAML spec.
+-    std::vector<std::string> r;
+-    for (const char *c = output; *c != '\0';) {
+-        const char *json = c;
+-        while (*c != '\0')
+-            ++c;
+-        ++c;
+-        r.emplace_back(json);
+-    }
+-    jsonnet_realloc(vm, output, 0);
+-    for (const auto &str : r) {
+-        (*o) << "---\n";
+-        (*o) << str;
+-    }
+-    if (r.size() > 0)
+-        (*o) << "...\n";
+-    o->flush();
+-
+-    if (output_file.empty()) {
+-        std::cout.flush();
+-    } else {
+-        f.close();
+-        if (!f.good()) {
+-            std::string msg = "Writing to output file: " + output_file;
+-            perror(msg.c_str());
+-            return false;
+-        }
+-    }
+-
 -    return true;
 -}
 -

oss-internship-2020/jsonnet/jsonnet_helper.cc

@@ -27,10 +27,18 @@ char* c_jsonnet_evaluate_snippet(struct JsonnetVm* vm, const char* filename,
   return jsonnet_evaluate_snippet(vm, filename, snippet, error);
 }
 
-char* c_jsonnet_evaluate_snippet_multi(struct JsonnetVm *vm, const char *filename, const char *snippet, int *error) {
+char* c_jsonnet_evaluate_snippet_multi(struct JsonnetVm* vm,
+                                       const char* filename,
+                                       const char* snippet, int* error) {
   return jsonnet_evaluate_snippet_multi(vm, filename, snippet, error);
 }
 
+char* c_jsonnet_evaluate_snippet_stream(struct JsonnetVm* vm,
+                                       const char* filename,
+                                       const char* snippet, int* error) {
+  return jsonnet_evaluate_snippet_stream(vm, filename, snippet, error);
+}
+
 char* c_read_input(bool filename_is_code, const char* filename) {
   std::string s_filename(filename);
   std::string s_input;
@@ -50,11 +58,16 @@ bool c_write_output_file(const char* output, const char* output_file) {
   return write_output_file(output, s_output_file);
 }
 
-bool c_write_multi_output_files(JsonnetVm *vm, char *output, char* output_dir) {
+bool c_write_multi_output_files(JsonnetVm* vm, char* output, char* output_dir) {
   std::string s_output_dir(output_dir);
   return write_multi_output_files(vm, output, s_output_dir);
 }
 
+bool c_write_output_stream(JsonnetVm* vm, char* output, char* output_file) {
+  std::string s_output_file(output_file);
+  return write_output_stream(vm, output, s_output_file);
+}
+
 char* c_jsonnet_realloc(JsonnetVm* vm, char* str, size_t sz) {
   return jsonnet_realloc(vm, str, sz);
 }

oss-internship-2020/jsonnet/jsonnet_helper.h

@@ -26,7 +26,15 @@ extern "C" char* c_jsonnet_evaluate_snippet(struct JsonnetVm* vm,
                                             const char* filename, char* snippet,
                                             int* error);
 
-extern "C" char* c_jsonnet_evaluate_snippet_multi(struct JsonnetVm *vm, const char *filename, const char *snippet, int *error);
+extern "C" char* c_jsonnet_evaluate_snippet_multi(struct JsonnetVm* vm,
+                                                  const char* filename,
+                                                  const char* snippet,
+                                                  int* error);
+
+extern "C" char* c_jsonnet_evaluate_snippet_stream(struct JsonnetVm* vm,
+                                                   const char* filename,
+                                                   const char* snippet,
+                                                   int* error);
 
 extern "C" char* c_read_input(bool filename_is_code, const char* filename);
 
@@ -37,6 +45,12 @@ extern "C" bool c_write_output_file(const char* output,
 
 extern "C" char* c_jsonnet_realloc(JsonnetVm* vm, char* str, size_t sz);
 
-extern "C" bool c_write_multi_output_files(JsonnetVm *vm, char *output, char* output_dir);
+extern "C" bool c_write_multi_output_files(JsonnetVm* vm, char* output,
+                                           char* output_dir);
 
-bool write_multi_output_files(JsonnetVm *vm, char *output, const std::string &output_dir);
+bool write_multi_output_files(JsonnetVm* vm, char* output,
+                              const std::string& output_dir);
+
+extern "C" bool c_write_output_stream(JsonnetVm* vm, char* output, char* output_file);
+
+bool write_output_stream(JsonnetVm* vm, char* output, const std::string& output_file);