Init guetzli sandbox

oss-internship-2020/guetzli/BUILD.bazel

@@ -0,0 +1,65 @@
+load("@rules_cc//cc:defs.bzl", "cc_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+load(
+    "@com_google_sandboxed_api//sandboxed_api/bazel:proto.bzl",
+    "sapi_proto_library",
+)
+load(
+    "@com_google_sandboxed_api//sandboxed_api/bazel:sapi.bzl",
+    "sapi_library",
+)
+
+cc_library(
+    name = "guetzli_wrapper",
+    srcs = ["guetzli_entry_points.cc"],
+    hdrs = ["guetzli_entry_points.h"],
+    deps = [
+        "@guetzli//:guetzli_lib",
+        "@com_google_sandboxed_api//sandboxed_api:lenval_core",
+        "@com_google_sandboxed_api//sandboxed_api:vars",
+        #"@com_google_sandboxed_api//sandboxed_api/sandbox2/util:temp_file", visibility error
+        "@png_archive//:png"
+    ],
+    visibility = ["//visibility:public"]
+)
+
+sapi_library(
+    name = "guetzli_sapi",
+    #srcs = ["guetzli_transaction.cc"], // Error when try to place definitions insde .cc file
+    hdrs = ["guetzli_sandbox.h", "guetzli_transaction.h"],
+    functions = [
+        "ProcessJPEGString",
+        "ProcessRGBData",
+        "ButteraugliScoreQuality",
+        "ReadPng",
+        "ReadJpegData",
+        "ReadDataFromFd",
+        "WriteDataToFd"
+    ],
+    input_files = ["guetzli_entry_points.h"],
+    lib = ":guetzli_wrapper",
+    lib_name = "Guetzli",
+    visibility = ["//visibility:public"],
+    namespace = "guetzli::sandbox"
+)
+
+# cc_library(
+#   name = "guetzli_sapi_transaction",
+#   #srcs = ["guetzli_transaction.cc"],
+#   hdrs = ["guetzli_transaction.h"],
+#   deps = [
+#     ":guetzli_sapi"
+#   ],
+#   visibility = ["//visibility:public"]
+# )
+
+cc_binary(
+    name="guetzli_sandboxed",
+    srcs=["guetzli_sandboxed.cc"],
+    includes = ["."],
+    visibility= [ "//visibility:public" ],
+    deps = [
+        #":guetzli_sapi_transaction"
+        ":guetzli_sapi"
+    ]
+)

oss-internship-2020/guetzli/WORKSPACE

@@ -0,0 +1,79 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+workspace(name = "guetzli_sandboxed")
+
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+
+# Include the Sandboxed API dependency if it does not already exist in this
+# project. This ensures that this workspace plays well with other external
+# dependencies that might use Sandboxed API.
+maybe(
+    git_repository,
+    name = "com_google_sandboxed_api",
+    # This example depends on the latest master. In an embedding project, it
+    # is advisable to pin Sandboxed API to a specific revision instead.
+    # commit = "ba47adc21d4c9bc316f3c7c32b0faaef952c111e",  # 2020-05-15
+    branch = "master",
+    remote = "https://github.com/google/sandboxed-api.git",
+)
+
+# From here on, Sandboxed API files are available. The statements below setup
+# transitive dependencies such as Abseil. Like above, those will only be
+# included if they don't already exist in the project.
+load(
+    "@com_google_sandboxed_api//sandboxed_api/bazel:sapi_deps.bzl",
+    "sapi_deps",
+)
+
+sapi_deps()
+
+# Need to separately setup Protobuf dependencies in order for the build rules
+# to work.
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+
+protobuf_deps()
+
+maybe(
+  git_repository,
+  name = "guetzli",
+  remote = "https://github.com/google/guetzli.git",
+  branch = "master"
+)
+
+maybe(
+  git_repository,
+  name = "googletest",
+  remote = "https://github.com/google/googletest",
+  tag = "release-1.10.0"
+)
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "png_archive",
+    build_file = "png.BUILD",
+    sha256 = "a941dc09ca00148fe7aaf4ecdd6a67579c293678ed1e1cf633b5ffc02f4f8cf7",
+    strip_prefix = "libpng-1.2.57",
+    url = "http://github.com/glennrp/libpng/archive/v1.2.57.zip",
+)
+
+http_archive(
+    name = "zlib_archive",
+    build_file = "zlib.BUILD",
+    sha256 = "8d7e9f698ce48787b6e1c67e6bff79e487303e66077e25cb9784ac8835978017",
+    strip_prefix = "zlib-1.2.10",
+    url = "http://zlib.net/fossils/zlib-1.2.10.tar.gz",
+)

oss-internship-2020/guetzli/external/png.BUILD

@@ -0,0 +1,33 @@
+# Description:
+#   libpng is the official PNG reference library.
+
+licenses(["notice"])  # BSD/MIT-like license
+
+cc_library(
+    name = "png",
+    srcs = [
+        "png.c",
+        "pngerror.c",
+        "pngget.c",
+        "pngmem.c",
+        "pngpread.c",
+        "pngread.c",
+        "pngrio.c",
+        "pngrtran.c",
+        "pngrutil.c",
+        "pngset.c",
+        "pngtrans.c",
+        "pngwio.c",
+        "pngwrite.c",
+        "pngwtran.c",
+        "pngwutil.c",
+    ],
+    hdrs = [
+        "png.h",
+        "pngconf.h",
+    ],
+    includes = ["."],
+    linkopts = ["-lm"],
+    visibility = ["//visibility:public"],
+    deps = ["@zlib_archive//:zlib"],
+)

oss-internship-2020/guetzli/external/zlib.BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # BSD/MIT-like license (for zlib)
+
+cc_library(
+    name = "zlib",
+    srcs = [
+        "adler32.c",
+        "compress.c",
+        "crc32.c",
+        "crc32.h",
+        "deflate.c",
+        "deflate.h",
+        "gzclose.c",
+        "gzguts.h",
+        "gzlib.c",
+        "gzread.c",
+        "gzwrite.c",
+        "infback.c",
+        "inffast.c",
+        "inffast.h",
+        "inffixed.h",
+        "inflate.c",
+        "inflate.h",
+        "inftrees.c",
+        "inftrees.h",
+        "trees.c",
+        "trees.h",
+        "uncompr.c",
+        "zconf.h",
+        "zutil.c",
+        "zutil.h",
+    ],
+    hdrs = ["zlib.h"],
+    includes = ["."],
+)

oss-internship-2020/guetzli/guetzli_entry_points.cc

@@ -0,0 +1,245 @@
+#include "guetzli/jpeg_data_reader.h"
+#include "guetzli/quality.h"
+#include "guetzli_entry_points.h"
+#include "png.h"
+#include "sandboxed_api/sandbox2/util/fileops.h"
+
+#include <algorithm>
+#include <iostream>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <vector>
+
+namespace {
+
+inline uint8_t BlendOnBlack(const uint8_t val, const uint8_t alpha) {
+  return (static_cast<int>(val) * static_cast<int>(alpha) + 128) / 255;
+}
+
+template<typename T>
+void CopyMemoryToLenVal(const T* data, size_t size, 
+                        sapi::LenValStruct* out_data) {
+  free(out_data->data); // Not sure about this
+  out_data->size = size;
+  T* new_out = static_cast<T*>(malloc(size));
+  memcpy(new_out, data, size);
+  out_data->data = new_out;
+}
+
+} // namespace
+
+extern "C" bool ProcessJPEGString(const guetzli::Params* params,
+                                  int verbose,
+                                  sapi::LenValStruct* in_data, 
+                                  sapi::LenValStruct* out_data)
+{
+  std::string in_data_temp(static_cast<const char*>(in_data->data), 
+      in_data->size);
+
+  guetzli::ProcessStats stats;
+  if (verbose > 0) {
+    stats.debug_output_file = stderr;
+  }
+
+  std::string temp_out = "";
+  auto result = guetzli::Process(*params, &stats, in_data_temp, &temp_out);
+
+  if (result) {
+    CopyMemoryToLenVal(temp_out.data(), temp_out.size(), out_data);
+  }
+
+  return result;
+}
+
+extern "C" bool ProcessRGBData(const guetzli::Params* params,
+                                int verbose,
+                                sapi::LenValStruct* rgb, 
+                                int w, int h,
+                                sapi::LenValStruct* out_data)
+{
+  std::vector<uint8_t> in_data_temp;
+  in_data_temp.reserve(rgb->size);
+
+  auto* rgb_data = static_cast<uint8_t*>(rgb->data);
+  std::copy(rgb_data, rgb_data + rgb->size, std::back_inserter(in_data_temp));
+
+  guetzli::ProcessStats stats;
+  if (verbose > 0) {
+    stats.debug_output_file = stderr;
+  }
+
+  std::string temp_out = "";
+  auto result = 
+      guetzli::Process(*params, &stats, in_data_temp, w, h, &temp_out);
+
+  //TODO: Move shared part of the code to another function
+  if (result) {
+    CopyMemoryToLenVal(temp_out.data(), temp_out.size(), out_data);
+  }
+
+  return result;
+}
+
+extern "C" bool ReadPng(sapi::LenValStruct* in_data, 
+                        int* xsize, int* ysize,
+                        sapi::LenValStruct* rgb_out)
+{
+  std::string data(static_cast<const char*>(in_data->data), in_data->size);
+  std::vector<uint8_t> rgb;
+  
+   png_structp png_ptr =
+      png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr);
+  if (!png_ptr) {
+    return false;
+  }
+
+  png_infop info_ptr = png_create_info_struct(png_ptr);
+  if (!info_ptr) {
+    png_destroy_read_struct(&png_ptr, nullptr, nullptr);
+    return false;
+  }
+
+  if (setjmp(png_jmpbuf(png_ptr)) != 0) {
+    // Ok we are here because of the setjmp.
+    png_destroy_read_struct(&png_ptr, &info_ptr, nullptr);
+    return false;
+  }
+
+  std::istringstream memstream(data, std::ios::in | std::ios::binary);
+  png_set_read_fn(png_ptr, static_cast<void*>(&memstream), [](png_structp png_ptr, png_bytep outBytes, png_size_t byteCountToRead) {
+    std::istringstream& memstream = *static_cast<std::istringstream*>(png_get_io_ptr(png_ptr));
+    
+    memstream.read(reinterpret_cast<char*>(outBytes), byteCountToRead);
+
+    if (memstream.eof()) png_error(png_ptr, "unexpected end of data");
+    if (memstream.fail()) png_error(png_ptr, "read from memory error");
+  });
+
+  // The png_transforms flags are as follows:
+  // packing == convert 1,2,4 bit images,
+  // strip == 16 -> 8 bits / channel,
+  // shift == use sBIT dynamics, and
+  // expand == palettes -> rgb, grayscale -> 8 bit images, tRNS -> alpha.
+  const unsigned int png_transforms =
+      PNG_TRANSFORM_PACKING | PNG_TRANSFORM_EXPAND | PNG_TRANSFORM_STRIP_16;
+
+  png_read_png(png_ptr, info_ptr, png_transforms, nullptr);
+
+  png_bytep* row_pointers = png_get_rows(png_ptr, info_ptr);
+
+  *xsize = png_get_image_width(png_ptr, info_ptr);
+  *ysize = png_get_image_height(png_ptr, info_ptr);
+  rgb.resize(3 * (*xsize) * (*ysize));
+
+  const int components = png_get_channels(png_ptr, info_ptr);
+  switch (components) {
+    case 1: {
+      // GRAYSCALE
+      for (int y = 0; y < *ysize; ++y) {
+        const uint8_t* row_in = row_pointers[y];
+        uint8_t* row_out = &(rgb)[3 * y * (*xsize)];
+        for (int x = 0; x < *xsize; ++x) {
+          const uint8_t gray = row_in[x];
+          row_out[3 * x + 0] = gray;
+          row_out[3 * x + 1] = gray;
+          row_out[3 * x + 2] = gray;
+        }
+      }
+      break;
+    }
+    case 2: {
+      // GRAYSCALE + ALPHA
+      for (int y = 0; y < *ysize; ++y) {
+        const uint8_t* row_in = row_pointers[y];
+        uint8_t* row_out = &(rgb)[3 * y * (*xsize)];
+        for (int x = 0; x < *xsize; ++x) {
+          const uint8_t gray = BlendOnBlack(row_in[2 * x], row_in[2 * x + 1]);
+          row_out[3 * x + 0] = gray;
+          row_out[3 * x + 1] = gray;
+          row_out[3 * x + 2] = gray;
+        }
+      }
+      break;
+    }
+    case 3: {
+      // RGB
+      for (int y = 0; y < *ysize; ++y) {
+        const uint8_t* row_in = row_pointers[y];
+        uint8_t* row_out = &(rgb)[3 * y * (*xsize)];
+        memcpy(row_out, row_in, 3 * (*xsize));
+      }
+      break;
+    }
+    case 4: {
+      // RGBA
+      for (int y = 0; y < *ysize; ++y) {
+        const uint8_t* row_in = row_pointers[y];
+        uint8_t* row_out = &(rgb)[3 * y * (*xsize)];
+        for (int x = 0; x < *xsize; ++x) {
+          const uint8_t alpha = row_in[4 * x + 3];
+          row_out[3 * x + 0] = BlendOnBlack(row_in[4 * x + 0], alpha);
+          row_out[3 * x + 1] = BlendOnBlack(row_in[4 * x + 1], alpha);
+          row_out[3 * x + 2] = BlendOnBlack(row_in[4 * x + 2], alpha);
+        }
+      }
+      break;
+    }
+    default:
+      png_destroy_read_struct(&png_ptr, &info_ptr, nullptr);
+      return false;
+  }
+  png_destroy_read_struct(&png_ptr, &info_ptr, nullptr);
+
+  CopyMemoryToLenVal(rgb.data(), rgb.size(), rgb_out);
+
+  return true;
+}
+
+extern "C" bool ReadJpegData(sapi::LenValStruct* in_data, 
+                              int mode,
+                              int* xsize, int* ysize)
+{
+  std::string data(static_cast<const char*>(in_data->data), in_data->size);
+  guetzli::JPEGData jpg;
+
+  auto result = guetzli::ReadJpeg(data, 
+    static_cast<guetzli::JpegReadMode>(mode), &jpg);
+
+  if (result) {
+    *xsize = jpg.width;
+    *ysize = jpg.height;
+  }
+
+  return result;
+}
+
+extern "C" double ButteraugliScoreQuality(double quality) {
+  return guetzli::ButteraugliScoreForQuality(quality);
+}
+
+extern "C" bool ReadDataFromFd(int fd, sapi::LenValStruct* out_data) {
+  struct stat file_data;
+  auto status = fstat(fd, &file_data);
+  
+  if (status < 0) {
+    return false;
+  }
+  
+  auto fsize = file_data.st_size;
+
+  std::unique_ptr<char[]> buf(new char[fsize]);
+  status = read(fd, buf.get(), fsize);
+
+  if (status < 0) {
+    return false;
+  }
+  
+  CopyMemoryToLenVal(buf.get(), fsize, out_data);
+
+  return true;
+}
+
+extern "C" bool WriteDataToFd(int fd, sapi::LenValStruct* data) {
+  return sandbox2::file_util::fileops::WriteToFD(fd, 
+    static_cast<const char*>(data->data), data->size);
+}

oss-internship-2020/guetzli/guetzli_entry_points.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "guetzli/processor.h"
+#include "sandboxed_api/lenval_core.h"
+#include "sandboxed_api/vars.h"
+
+extern "C" bool ProcessJPEGString(const guetzli::Params* params,
+                                  int verbose,
+                                  sapi::LenValStruct* in_data, 
+                                  sapi::LenValStruct* out_data);
+
+extern "C" bool ProcessRGBData(const guetzli::Params* params,
+                                int verbose,
+                                sapi::LenValStruct* rgb, 
+                                int w, int h,
+                                sapi::LenValStruct* out_data);
+
+extern "C" bool ReadPng(sapi::LenValStruct* in_data, 
+                        int* xsize, int* ysize,
+                        sapi::LenValStruct* rgb_out);
+
+extern "C" bool ReadJpegData(sapi::LenValStruct* in_data, 
+                              int mode, int* xsize, int* ysize);
+
+extern "C" double ButteraugliScoreQuality(double quality);
+
+extern "C" bool ReadDataFromFd(int fd, sapi::LenValStruct* out_data);
+
+extern "C" bool WriteDataToFd(int fd, sapi::LenValStruct* data);

oss-internship-2020/guetzli/guetzli_sandbox.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <libgen.h>
+#include <syscall.h>
+
+#include "guetzli_sapi.sapi.h"
+#include "sandboxed_api/sandbox2/policy.h"
+#include "sandboxed_api/sandbox2/policybuilder.h"
+#include "sandboxed_api/util/flag.h"
+
+namespace guetzli {
+namespace sandbox {
+
+class GuetzliSapiSandbox : public GuetzliSandbox {
+  public:
+  std::unique_ptr<sandbox2::Policy> ModifyPolicy(
+      sandbox2::PolicyBuilder*) override {
+
+      return sandbox2::PolicyBuilder()
+        .AllowStaticStartup()
+        .AllowRead()
+        .AllowSystemMalloc()
+        .AllowWrite()
+        .AllowExit()
+        .AllowStat()
+        .AllowSyscalls({
+          __NR_futex,
+          __NR_close,
+          __NR_recvmsg // Seems like this one needed to work with remote file descriptors
+        })
+        .BuildOrDie();
+    }
+};
+
+} // namespace sandbox
+} // namespace guetzli

oss-internship-2020/guetzli/guetzli_sandboxed.cc

@@ -0,0 +1,160 @@
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include <iostream>
+
+#include "guetzli_transaction.h"
+#include "sandboxed_api/sandbox2/util/fileops.h"
+#include "sandboxed_api/util/statusor.h"
+
+namespace {
+
+constexpr int kDefaultJPEGQuality = 95;
+constexpr int kDefaultMemlimitMB = 6000; // in MB
+//constexpr absl::string_view kMktempSuffix = "XXXXXX";
+
+// sapi::StatusOr<std::pair<std::string, int>> CreateNamedTempFile(
+//     absl::string_view prefix) {
+//   std::string name_template = absl::StrCat(prefix, kMktempSuffix);
+//   int fd = mkstemp(&name_template[0]);
+//   if (fd < 0) {
+//     return absl::UnknownError("Error creating temp file");
+//   }
+//   return std::pair<std::string, int>{std::move(name_template), fd};
+// }
+
+void TerminateHandler() {
+  fprintf(stderr, "Unhandled exception. Most likely insufficient memory available.\n"
+          "Make sure that there is 300MB/MPix of memory available.\n");
+  exit(1);
+}
+
+void Usage() {
+  fprintf(stderr,
+      "Guetzli JPEG compressor. Usage: \n"
+      "guetzli [flags] input_filename output_filename\n"
+      "\n"
+      "Flags:\n"
+      "  --verbose    - Print a verbose trace of all attempts to standard output.\n"
+      "  --quality Q  - Visual quality to aim for, expressed as a JPEG quality value.\n"
+      "                 Default value is %d.\n"
+      "  --memlimit M - Memory limit in MB. Guetzli will fail if unable to stay under\n"
+      "                 the limit. Default limit is %d MB.\n"
+      "  --nomemlimit - Do not limit memory usage.\n", kDefaultJPEGQuality, kDefaultMemlimitMB);
+  exit(1);
+}
+
+}  // namespace
+
+int main(int argc, const char** argv) {
+  std::set_terminate(TerminateHandler);
+
+  int verbose = 0;
+  int quality = kDefaultJPEGQuality;
+  int memlimit_mb = kDefaultMemlimitMB;   
+
+  int opt_idx = 1;
+  for(;opt_idx < argc;opt_idx++) {
+    if (strnlen(argv[opt_idx], 2) < 2 || argv[opt_idx][0] != '-' || argv[opt_idx][1] != '-')
+      break;
+
+    if (!strcmp(argv[opt_idx], "--verbose")) {
+      verbose = 1;
+    } else if (!strcmp(argv[opt_idx], "--quality")) {
+      opt_idx++;
+      if (opt_idx >= argc)
+        Usage();
+      quality = atoi(argv[opt_idx]);
+    } else if (!strcmp(argv[opt_idx], "--memlimit")) {
+      opt_idx++;
+      if (opt_idx >= argc)
+        Usage();
+      memlimit_mb = atoi(argv[opt_idx]);
+    } else if (!strcmp(argv[opt_idx], "--nomemlimit")) {
+      memlimit_mb = -1;
+    } else if (!strcmp(argv[opt_idx], "--")) {
+      opt_idx++;
+      break;
+    } else {
+      fprintf(stderr, "Unknown commandline flag: %s\n", argv[opt_idx]);
+      Usage();
+    }
+  }
+
+  if (argc - opt_idx != 2) {
+    Usage();
+  }
+
+  sandbox2::file_util::fileops::FDCloser in_fd_closer(
+    open(argv[opt_idx], O_RDONLY));
+  
+  if (in_fd_closer.get() < 0) {
+    fprintf(stderr, "Can't open input file: %s\n", argv[opt_idx]);
+    return 1;
+  }
+
+  // auto out_temp_file = CreateNamedTempFile("/tmp/" + std::string(argv[opt_idx + 1]));
+  // if (!out_temp_file.ok()) {
+  //   fprintf(stderr, "Can't create temporary output file: %s\n", 
+  //     argv[opt_idx + 1]);
+  //   return 1;
+  // }
+  // sandbox2::file_util::fileops::FDCloser out_fd_closer(
+  //     out_temp_file.value().second);
+
+  // if (unlink(out_temp_file.value().first.c_str()) < 0) {
+  //   fprintf(stderr, "Error unlinking temp out file: %s\n", 
+  //       out_temp_file.value().first.c_str());
+  //   return 1;
+  // }
+
+  sandbox2::file_util::fileops::FDCloser out_fd_closer(
+    open(".", O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR));
+
+  if (out_fd_closer.get() < 0) {
+    fprintf(stderr, "Can't create temporary output file: %s\n", argv[opt_idx]);
+    return 1;
+  }
+
+  // sandbox2::file_util::fileops::FDCloser out_fd_closer(open(argv[opt_idx + 1], 
+  //   O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP));
+  
+  // if (out_fd_closer.get() < 0) {
+  //   fprintf(stderr, "Can't open output file: %s\n", argv[opt_idx + 1]);
+  //   return 1;
+  // }
+
+  guetzli::sandbox::TransactionParams params = {
+    in_fd_closer.get(),
+    out_fd_closer.get(),
+    verbose,
+    quality,
+    memlimit_mb
+  };
+
+  guetzli::sandbox::GuetzliTransaction transaction(std::move(params));
+  auto result = transaction.Run();
+
+  if (result.ok()) {
+    if (access(argv[opt_idx + 1], F_OK) != -1) {
+      if (remove(argv[opt_idx + 1]) < 0) {
+        fprintf(stderr, "Error deleting existing output file: %s\n", 
+          argv[opt_idx + 1]);
+      }
+    } 
+
+    std::stringstream path;
+    path << "/proc/self/fd/" << out_fd_closer.get();
+    linkat(AT_FDCWD, path.str().c_str(), AT_FDCWD, argv[opt_idx + 1],
+                                          AT_SYMLINK_FOLLOW);
+  }
+  else {
+    fprintf(stderr, "%s\n", result.ToString().c_str()); // Use cerr instead ?
+    return 1;
+  }
+
+  return 0;
+}

oss-internship-2020/guetzli/guetzli_transaction.cc

@@ -0,0 +1,160 @@
+#include "guetzli_transaction.h"
+
+#include <iostream>
+
+namespace guetzli {
+namespace sandbox {
+
+absl::Status GuetzliTransaction::Init() {
+  SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&in_fd_));
+  SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&out_fd_));
+
+  if (in_fd_.GetRemoteFd() < 0) {
+    return absl::FailedPreconditionError(
+        "Error receiving remote FD: remote input fd is set to -1");
+  }
+  if (out_fd_.GetRemoteFd() < 0) {
+    return absl::FailedPreconditionError(
+        "Error receiving remote FD: remote output fd is set to -1");
+  }
+
+  return absl::OkStatus();
+}
+
+  absl::Status GuetzliTransaction::ProcessPng(GuetzliAPi* api, 
+                                              sapi::v::Struct<Params>* params, 
+                                              sapi::v::LenVal* input, 
+                                              sapi::v::LenVal* output) const {
+    sapi::v::Int xsize;
+    sapi::v::Int ysize;
+    sapi::v::LenVal rgb_in(0);
+
+    auto read_result = api->ReadPng(input->PtrBefore(), xsize.PtrBoth(), 
+      ysize.PtrBoth(), rgb_in.PtrBoth());
+      
+    if (!read_result.value_or(false)) {
+      return absl::FailedPreconditionError(
+        "Error reading PNG data from input file"
+      );
+    }
+
+    double pixels = static_cast<double>(xsize.GetValue()) * ysize.GetValue();
+    if (params_.memlimit_mb != -1
+        && (pixels * kBytesPerPixel / (1 << 20) > params_.memlimit_mb
+            || params_.memlimit_mb < kLowestMemusageMB)) {
+      return absl::FailedPreconditionError(
+        "Memory limit would be exceeded"
+      );
+    }
+
+    auto result = api->ProcessRGBData(params->PtrBefore(), params_.verbose, 
+                                    rgb_in.PtrBefore(), xsize.GetValue(), 
+                                    ysize.GetValue(), output->PtrBoth());
+    if (!result.value_or(false)) {
+      return absl::FailedPreconditionError(
+        "Guetzli processing failed"
+      );
+    }
+
+    return absl::OkStatus();
+  }
+
+  absl::Status GuetzliTransaction::ProcessJpeg(GuetzliApi* api, 
+                                              sapi::v::Struct<Params>* params, 
+                                              sapi::v::LenVal* input, 
+                                              sapi::v::LenVal* output) const {
+    ::sapi::v::Int xsize;
+    ::sapi::v::Int ysize;
+    auto read_result = api->ReadJpegData(input->PtrBefore(), 0, xsize.PtrBoth(), 
+      ysize.PtrBoth());
+
+    if (!read_result.value_or(false)) {
+      return absl::FailedPreconditionError(
+        "Error reading JPG data from input file"
+      );
+    }
+
+    double pixels = static_cast<double>(xsize.GetValue()) * ysize.GetValue();
+    if (params_.memlimit_mb != -1
+        && (pixels * kBytesPerPixel / (1 << 20) > params_.memlimit_mb
+            || params_.memlimit_mb < kLowestMemusageMB)) {
+      return absl::FailedPreconditionError(
+        "Memory limit would be exceeded"
+      );
+    }
+
+    auto result = api->ProcessJPEGString(params->PtrBefore(), params_.verbose, 
+      input->PtrBefore(), output->PtrBoth());
+
+    if (!result.value_or(false)) {
+      return absl::FailedPreconditionError(
+        "Guetzli processing failed"
+      );
+    }
+
+    return absl::OkStatus();
+  }
+
+absl::Status GuetzliTransaction::Main() {
+  GuetzliApi api(sandbox());
+
+  sapi::v::LenVal input(0);
+  sapi::v::LenVal output(0);
+  sapi::v::Struct<Params> params;
+  
+  auto read_result = api.ReadDataFromFd(in_fd_.GetRemoteFd(), input.PtrBoth());
+
+  if (!read_result.value_or(false)) {
+    return absl::FailedPreconditionError(
+      "Error reading data inside sandbox"
+    );
+  }
+
+  auto score_quality_result = api.ButteraugliScoreQuality(params_.quality);
+
+  if (!score_quality_result.ok()) {
+    return absl::FailedPreconditionError(
+      "Error calculating butteraugli score"
+    );
+  }
+
+  params.mutable_data()->butteraugli_target = score_quality_result.value();
+
+  static const unsigned char kPNGMagicBytes[] = {
+    0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n',
+  };
+  
+  if (input.GetDataSize() >= 8 &&
+    memcmp(input.GetData(), kPNGMagicBytes, sizeof(kPNGMagicBytes)) == 0) {
+    auto process_status = ProcessPng(&api, &params, &input, &output);
+
+    if (!process_status.ok()) {
+      return process_status;
+    }
+  } else {
+    auto process_status = ProcessJpeg(&api, &params, &input, &output);
+
+    if (!process_status.ok()) {
+      return process_status;
+    }
+  }
+
+  auto write_result = api.WriteDataToFd(out_fd_.GetRemoteFd(), 
+    output.PtrBefore());
+
+  if (!write_result.value_or(false)) {
+    return absl::FailedPreconditionError(
+      "Error writing file inside sandbox"
+    );
+  }
+
+  return absl::OkStatus();
+}
+
+time_t GuetzliTransaction::CalculateTimeLimitFromImageSize(
+    uint64_t pixels) const {
+  return (pixels / kMpixPixels + 5) * 60;
+}
+
+} // namespace sandbox
+} // namespace guetzli

oss-internship-2020/guetzli/guetzli_transaction.h

@@ -0,0 +1,216 @@
+#pragma once
+
+#include <libgen.h>
+#include <syscall.h>
+
+#include "guetzli_sandbox.h"
+#include "sandboxed_api/transaction.h"
+#include "sandboxed_api/vars.h"
+
+namespace guetzli {
+namespace sandbox {
+
+constexpr int kDefaultTransactionRetryCount = 1;
+constexpr uint64_t kMpixPixels = 1'000'000;
+
+constexpr int kBytesPerPixel = 350;
+constexpr int kLowestMemusageMB = 100; // in MB
+
+struct TransactionParams {
+  int in_fd;
+  int out_fd;
+  int verbose;
+  int quality;
+  int memlimit_mb;
+};
+
+//Add optional time limit/retry count as a constructors arguments
+//Use differenet status errors
+class GuetzliTransaction : public sapi::Transaction {
+ public:
+  GuetzliTransaction(TransactionParams&& params)
+      : sapi::Transaction(std::make_unique<GuetzliSapiSandbox>())
+      , params_(std::move(params))
+      , in_fd_(params_.in_fd)
+      , out_fd_(params_.out_fd)
+  {
+    sapi::Transaction::set_retry_count(kDefaultTransactionRetryCount);
+    sapi::Transaction::SetTimeLimit(0);
+  }
+
+ private:
+  absl::Status Init() override;
+  absl::Status Main() final;
+
+  absl::Status ProcessPng(GuetzliApi* api, 
+                          sapi::v::Struct<Params>* params, 
+                          sapi::v::LenVal* input, 
+                          sapi::v::LenVal* output) const;
+
+  absl::Status ProcessJpeg(GuetzliApi* api, 
+                          sapi::v::Struct<Params>* params, 
+                          sapi::v::LenVal* input, 
+                          sapi::v::LenVal* output) const;
+
+  // As guetzli takes roughly 1 minute of CPU per 1 MPix we need to calculate 
+  // approximate time for transaction to complete
+  time_t CalculateTimeLimitFromImageSize(uint64_t pixels) const;
+
+  const TransactionParams params_;
+  sapi::v::Fd in_fd_;
+  sapi::v::Fd out_fd_;
+};
+
+absl::Status GuetzliTransaction::Init() {
+  SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&in_fd_));
+  SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&out_fd_));
+
+  if (in_fd_.GetRemoteFd() < 0) {
+    return absl::FailedPreconditionError(
+        "Error receiving remote FD: remote input fd is set to -1");
+  }
+  if (out_fd_.GetRemoteFd() < 0) {
+    return absl::FailedPreconditionError(
+        "Error receiving remote FD: remote output fd is set to -1");
+  }
+
+  return absl::OkStatus();
+}
+
+  absl::Status GuetzliTransaction::ProcessPng(GuetzliApi* api, 
+                                              sapi::v::Struct<Params>* params, 
+                                              sapi::v::LenVal* input, 
+                                              sapi::v::LenVal* output) const {
+    sapi::v::Int xsize;
+    sapi::v::Int ysize;
+    sapi::v::LenVal rgb_in(0);
+
+    auto read_result = api->ReadPng(input->PtrBefore(), xsize.PtrBoth(), 
+      ysize.PtrBoth(), rgb_in.PtrBoth());
+      
+    if (!read_result.value_or(false)) {
+      return absl::FailedPreconditionError(
+        "Error reading PNG data from input file"
+      );
+    }
+
+    double pixels = static_cast<double>(xsize.GetValue()) * ysize.GetValue();
+    if (params_.memlimit_mb != -1
+        && (pixels * kBytesPerPixel / (1 << 20) > params_.memlimit_mb
+            || params_.memlimit_mb < kLowestMemusageMB)) {
+      return absl::FailedPreconditionError(
+        "Memory limit would be exceeded"
+      );
+    }
+
+    auto result = api->ProcessRGBData(params->PtrBefore(), params_.verbose, 
+                                    rgb_in.PtrBefore(), xsize.GetValue(), 
+                                    ysize.GetValue(), output->PtrBoth());
+    if (!result.value_or(false)) {
+      return absl::FailedPreconditionError(
+        "Guetzli processing failed"
+      );
+    }
+
+    return absl::OkStatus();
+  }
+
+  absl::Status GuetzliTransaction::ProcessJpeg(GuetzliApi* api, 
+                                              sapi::v::Struct<Params>* params, 
+                                              sapi::v::LenVal* input, 
+                                              sapi::v::LenVal* output) const {
+    sapi::v::Int xsize;
+    sapi::v::Int ysize;
+    auto read_result = api->ReadJpegData(input->PtrBefore(), 0, xsize.PtrBoth(), 
+      ysize.PtrBoth());
+
+    if (!read_result.value_or(false)) {
+      return absl::FailedPreconditionError(
+        "Error reading JPG data from input file"
+      );
+    }
+
+    double pixels = static_cast<double>(xsize.GetValue()) * ysize.GetValue();
+    if (params_.memlimit_mb != -1
+        && (pixels * kBytesPerPixel / (1 << 20) > params_.memlimit_mb
+            || params_.memlimit_mb < kLowestMemusageMB)) {
+      return absl::FailedPreconditionError(
+        "Memory limit would be exceeded"
+      );
+    }
+
+    auto result = api->ProcessJPEGString(params->PtrBefore(), params_.verbose, 
+      input->PtrBefore(), output->PtrBoth());
+
+    if (!result.value_or(false)) {
+      return absl::FailedPreconditionError(
+        "Guetzli processing failed"
+      );
+    }
+
+    return absl::OkStatus();
+  }
+
+absl::Status GuetzliTransaction::Main() {
+  GuetzliApi api(sandbox());
+
+  sapi::v::LenVal input(0);
+  sapi::v::LenVal output(0);
+  sapi::v::Struct<Params> params;
+  
+  auto read_result = api.ReadDataFromFd(in_fd_.GetRemoteFd(), input.PtrBoth());
+
+  if (!read_result.value_or(false)) {
+    return absl::FailedPreconditionError(
+      "Error reading data inside sandbox"
+    );
+  }
+
+  auto score_quality_result = api.ButteraugliScoreQuality(params_.quality);
+
+  if (!score_quality_result.ok()) {
+    return absl::FailedPreconditionError(
+      "Error calculating butteraugli score"
+    );
+  }
+
+  params.mutable_data()->butteraugli_target = score_quality_result.value();
+
+  static const unsigned char kPNGMagicBytes[] = {
+    0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n',
+  };
+  
+  if (input.GetDataSize() >= 8 &&
+    memcmp(input.GetData(), kPNGMagicBytes, sizeof(kPNGMagicBytes)) == 0) {
+    auto process_status = ProcessPng(&api, &params, &input, &output);
+
+    if (!process_status.ok()) {
+      return process_status;
+    }
+  } else {
+    auto process_status = ProcessJpeg(&api, &params, &input, &output);
+
+    if (!process_status.ok()) {
+      return process_status;
+    }
+  }
+
+  auto write_result = api.WriteDataToFd(out_fd_.GetRemoteFd(), 
+    output.PtrBefore());
+
+  if (!write_result.value_or(false)) {
+    return absl::FailedPreconditionError(
+      "Error writing file inside sandbox"
+    );
+  }
+
+  return absl::OkStatus();
+}
+
+time_t GuetzliTransaction::CalculateTimeLimitFromImageSize(
+    uint64_t pixels) const {
+  return (pixels / kMpixPixels + 5) * 60;
+}
+
+} // namespace sandbox
+} // namespace guetzli

oss-internship-2020/guetzli/tests/BUILD.bazel

@@ -0,0 +1,22 @@
+# cc_test(
+#     name = "transaction_tests",
+#     srcs = ["guetzli_transaction_test.cc"],
+#     visibility=["//visibility:public"],
+#     includes = ["."],
+#     deps = [
+#       "//:guetzli_sapi",
+#       "@googletest//:gtest_main"
+#     ],
+# )
+
+# cc_test(
+#     name = "sapi_lib_tests",
+#     srcs = ["guetzli_sapi_test.cc"],
+#     visibility=["//visibility:public"],
+#     includes=[".."],
+#     deps = [
+#       "//:guetzli_sapi",
+#       "@googletest//:gtest_main"
+#     ],
+#     data = glob(["testdata/*"])
+# )

oss-internship-2020/guetzli/tests/guetzli_sapi_test.cc

@@ -0,0 +1,165 @@
+#include "gtest/gtest.h"
+#include "guetzli_sandbox.h"
+#include "sandboxed_api/sandbox2/util/fileops.h"
+#include "sandboxed_api/vars.h"
+
+#include <fstream>
+#include <memory>
+#include <syscall.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sstream>
+
+namespace guetzli {
+namespace sandbox {
+namespace tests {
+
+namespace {
+
+constexpr const char* IN_PNG_FILENAME = "bees.png";
+constexpr const char* IN_JPG_FILENAME = "landscape.jpg";
+
+constexpr int IN_PNG_FILE_SIZE = 177'424;
+constexpr int IN_JPG_FILE_SIZE = 14'418;
+
+constexpr int DEFAULT_QUALITY_TARGET = 95;
+
+constexpr const char* RELATIVE_PATH_TO_TESTDATA =
+  "/guetzli/guetzli-sandboxed/tests/testdata/";
+
+std::string GetPathToInputFile(const char* filename) {
+  return std::string(getenv("TEST_SRCDIR")) 
+    + std::string(RELATIVE_PATH_TO_TESTDATA)
+    + std::string(filename);
+}
+
+std::string ReadFromFile(const std::string& filename) {
+  std::ifstream stream(filename, std::ios::binary);
+
+  if (!stream.is_open()) {
+    return "";
+  }
+
+  std::stringstream result;
+  result << stream.rdbuf();
+  return result.str();
+}
+
+} // namespace
+
+class GuetzliSapiTest : public ::testing::Test {
+protected:
+  void SetUp() override {
+    sandbox_ = std::make_unique<GuetzliSapiSandbox>();
+    sandbox_->Init().IgnoreError();
+    api_ = std::make_unique<GuetzliApi>(sandbox_.get());
+  }
+  
+  std::unique_ptr<GuetzliSapiSandbox> sandbox_;
+  std::unique_ptr<GuetzliApi> api_;
+};
+
+TEST_F(GuetzliSapiTest, ReadDataFromFd) {
+  std::string input_file_path = GetPathToInputFile(IN_PNG_FILENAME);
+  int fd = open(input_file_path.c_str(), O_RDONLY);
+  ASSERT_TRUE(fd != -1) << "Error opening input file";
+  sapi::v::Fd remote_fd(fd);
+  auto send_fd_status = sandbox_->TransferToSandboxee(&remote_fd);
+  ASSERT_TRUE(send_fd_status.ok()) << "Error sending fd to sandboxee";
+  ASSERT_TRUE(remote_fd.GetRemoteFd() != -1) << "Error opening remote fd";
+  sapi::v::LenVal data(0);
+  auto read_status = 
+    api_->ReadDataFromFd(remote_fd.GetRemoteFd(), data.PtrBoth());
+  ASSERT_TRUE(read_status.value_or(false)) << "Error reading data from fd";
+  ASSERT_EQ(data.GetDataSize(), IN_PNG_FILE_SIZE) << "Wrong size of file";
+}
+
+// TEST_F(GuetzliSapiTest, WriteDataToFd) {
+
+// }
+
+TEST_F(GuetzliSapiTest, ReadPng) {
+  std::string data = ReadFromFile(GetPathToInputFile(IN_PNG_FILENAME));
+  ASSERT_EQ(data.size(), IN_PNG_FILE_SIZE) << "Error reading input file";
+  sapi::v::LenVal in_data(data.data(), data.size());
+  sapi::v::Int xsize, ysize;
+  sapi::v::LenVal rgb_out(0);
+
+  auto status = api_->ReadPng(in_data.PtrBefore(), xsize.PtrBoth(), 
+    ysize.PtrBoth(), rgb_out.PtrBoth());
+  ASSERT_TRUE(status.value_or(false)) << "Error processing png data";
+  ASSERT_EQ(xsize.GetValue(), 444) << "Error parsing width";
+  ASSERT_EQ(ysize.GetValue(), 258) << "Error parsing height";
+}
+
+TEST_F(GuetzliSapiTest, ReadJpeg) {
+  std::string data = ReadFromFile(GetPathToInputFile(IN_JPG_FILENAME));
+  ASSERT_EQ(data.size(), IN_JPG_FILE_SIZE) << "Error reading input file";
+  sapi::v::LenVal in_data(data.data(), data.size());
+  sapi::v::Int xsize, ysize;
+
+  auto status = api_->ReadJpegData(in_data.PtrBefore(), 0, 
+    xsize.PtrBoth(), ysize.PtrBoth());
+  ASSERT_TRUE(status.value_or(false)) << "Error processing jpeg data";
+  ASSERT_EQ(xsize.GetValue(), 180) << "Error parsing width";
+  ASSERT_EQ(ysize.GetValue(), 180) << "Error parsing height";
+}
+
+// This test can take up to few minutes depending on your hardware
+TEST_F(GuetzliSapiTest, ProcessRGB) {
+  std::string data = ReadFromFile(GetPathToInputFile(IN_PNG_FILENAME));
+  ASSERT_EQ(data.size(), IN_PNG_FILE_SIZE) << "Error reading input file";
+  sapi::v::LenVal in_data(data.data(), data.size());
+  sapi::v::Int xsize, ysize;
+  sapi::v::LenVal rgb_out(0);
+
+  auto status = api_->ReadPng(in_data.PtrBefore(), xsize.PtrBoth(), 
+    ysize.PtrBoth(), rgb_out.PtrBoth());
+  ASSERT_TRUE(status.value_or(false)) << "Error processing png data";
+  ASSERT_EQ(xsize.GetValue(), 444) << "Error parsing width";
+  ASSERT_EQ(ysize.GetValue(), 258) << "Error parsing height";
+  auto quality = 
+    api_->ButteraugliScoreQuality(static_cast<double>(DEFAULT_QUALITY_TARGET));
+  ASSERT_TRUE(quality.ok()) << "Error calculating butteraugli quality";
+  sapi::v::Struct<Params> params;
+  sapi::v::LenVal out_data(0);
+  params.mutable_data()->butteraugli_target = quality.value();
+
+  status = api_->ProcessRGBData(params.PtrBefore(), 0, rgb_out.PtrBefore(), 
+    xsize.GetValue(), ysize.GetValue(), out_data.PtrBoth());
+  ASSERT_TRUE(status.value_or(false)) << "Error processing png file";
+  ASSERT_EQ(out_data.GetDataSize(), 38'625);
+  //ADD COMPARSION WITH REFERENCE OUTPUT
+}
+
+// This test can take up to few minutes depending on your hardware
+TEST_F(GuetzliSapiTest, ProcessJpeg) {
+  std::string data = ReadFromFile(GetPathToInputFile(IN_JPG_FILENAME));
+  ASSERT_EQ(data.size(), IN_JPG_FILE_SIZE) << "Error reading input file";
+  sapi::v::LenVal in_data(data.data(), data.size());
+  sapi::v::Int xsize, ysize;
+
+  auto status = api_->ReadJpegData(in_data.PtrBefore(), 0, 
+    xsize.PtrBoth(), ysize.PtrBoth());
+  ASSERT_TRUE(status.value_or(false)) << "Error processing jpeg data";
+  ASSERT_EQ(xsize.GetValue(), 180) << "Error parsing width";
+  ASSERT_EQ(ysize.GetValue(), 180) << "Error parsing height";
+
+  auto quality = 
+    api_->ButteraugliScoreQuality(static_cast<double>(DEFAULT_QUALITY_TARGET));
+  ASSERT_TRUE(quality.ok()) << "Error calculating butteraugli quality";
+  sapi::v::Struct<Params> params;
+  params.mutable_data()->butteraugli_target = quality.value();
+  sapi::v::LenVal out_data(0);
+
+  status = api_->ProcessJPEGString(params.PtrBefore(), 0, in_data.PtrBefore(), 
+    out_data.PtrBoth());
+  ASSERT_TRUE(status.value_or(false)) << "Error processing jpeg file";
+  ASSERT_EQ(out_data.GetDataSize(), 10'816);
+  //ADD COMPARSION WITH REFERENCE OUTPUT
+}
+
+} // namespace tests
+} // namespace sandbox
+} // namespace guetzli

oss-internship-2020/guetzli/tests/guetzli_transaction_test.cc

@@ -0,0 +1,2 @@
+#include "gtest/gtest.h"
+#include "guetzli_transaction.h"