diff --git a/oss-internship-2020/guetzli/BUILD.bazel b/oss-internship-2020/guetzli/BUILD.bazel new file mode 100644 index 0000000..345f879 --- /dev/null +++ b/oss-internship-2020/guetzli/BUILD.bazel @@ -0,0 +1,65 @@ +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load( + "@com_google_sandboxed_api//sandboxed_api/bazel:proto.bzl", + "sapi_proto_library", +) +load( + "@com_google_sandboxed_api//sandboxed_api/bazel:sapi.bzl", + "sapi_library", +) + +cc_library( + name = "guetzli_wrapper", + srcs = ["guetzli_entry_points.cc"], + hdrs = ["guetzli_entry_points.h"], + deps = [ + "@guetzli//:guetzli_lib", + "@com_google_sandboxed_api//sandboxed_api:lenval_core", + "@com_google_sandboxed_api//sandboxed_api:vars", + #"@com_google_sandboxed_api//sandboxed_api/sandbox2/util:temp_file", visibility error + "@png_archive//:png" + ], + visibility = ["//visibility:public"] +) + +sapi_library( + name = "guetzli_sapi", + #srcs = ["guetzli_transaction.cc"], // Error when try to place definitions insde .cc file + hdrs = ["guetzli_sandbox.h", "guetzli_transaction.h"], + functions = [ + "ProcessJPEGString", + "ProcessRGBData", + "ButteraugliScoreQuality", + "ReadPng", + "ReadJpegData", + "ReadDataFromFd", + "WriteDataToFd" + ], + input_files = ["guetzli_entry_points.h"], + lib = ":guetzli_wrapper", + lib_name = "Guetzli", + visibility = ["//visibility:public"], + namespace = "guetzli::sandbox" +) + +# cc_library( +# name = "guetzli_sapi_transaction", +# #srcs = ["guetzli_transaction.cc"], +# hdrs = ["guetzli_transaction.h"], +# deps = [ +# ":guetzli_sapi" +# ], +# visibility = ["//visibility:public"] +# ) + +cc_binary( + name="guetzli_sandboxed", + srcs=["guetzli_sandboxed.cc"], + includes = ["."], + visibility= [ "//visibility:public" ], + deps = [ + #":guetzli_sapi_transaction" + ":guetzli_sapi" + ] +) \ No newline at end of file diff --git a/oss-internship-2020/guetzli/WORKSPACE b/oss-internship-2020/guetzli/WORKSPACE new file mode 100644 index 0000000..17887de --- /dev/null +++ b/oss-internship-2020/guetzli/WORKSPACE @@ -0,0 +1,79 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +workspace(name = "guetzli_sandboxed") + +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +# Include the Sandboxed API dependency if it does not already exist in this +# project. This ensures that this workspace plays well with other external +# dependencies that might use Sandboxed API. +maybe( + git_repository, + name = "com_google_sandboxed_api", + # This example depends on the latest master. In an embedding project, it + # is advisable to pin Sandboxed API to a specific revision instead. + # commit = "ba47adc21d4c9bc316f3c7c32b0faaef952c111e", # 2020-05-15 + branch = "master", + remote = "https://github.com/google/sandboxed-api.git", +) + +# From here on, Sandboxed API files are available. The statements below setup +# transitive dependencies such as Abseil. Like above, those will only be +# included if they don't already exist in the project. +load( + "@com_google_sandboxed_api//sandboxed_api/bazel:sapi_deps.bzl", + "sapi_deps", +) + +sapi_deps() + +# Need to separately setup Protobuf dependencies in order for the build rules +# to work. +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") + +protobuf_deps() + +maybe( + git_repository, + name = "guetzli", + remote = "https://github.com/google/guetzli.git", + branch = "master" +) + +maybe( + git_repository, + name = "googletest", + remote = "https://github.com/google/googletest", + tag = "release-1.10.0" +) + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "png_archive", + build_file = "png.BUILD", + sha256 = "a941dc09ca00148fe7aaf4ecdd6a67579c293678ed1e1cf633b5ffc02f4f8cf7", + strip_prefix = "libpng-1.2.57", + url = "http://github.com/glennrp/libpng/archive/v1.2.57.zip", +) + +http_archive( + name = "zlib_archive", + build_file = "zlib.BUILD", + sha256 = "8d7e9f698ce48787b6e1c67e6bff79e487303e66077e25cb9784ac8835978017", + strip_prefix = "zlib-1.2.10", + url = "http://zlib.net/fossils/zlib-1.2.10.tar.gz", +) \ No newline at end of file diff --git a/oss-internship-2020/guetzli/external/png.BUILD b/oss-internship-2020/guetzli/external/png.BUILD new file mode 100644 index 0000000..9ff982b --- /dev/null +++ b/oss-internship-2020/guetzli/external/png.BUILD @@ -0,0 +1,33 @@ +# Description: +# libpng is the official PNG reference library. + +licenses(["notice"]) # BSD/MIT-like license + +cc_library( + name = "png", + srcs = [ + "png.c", + "pngerror.c", + "pngget.c", + "pngmem.c", + "pngpread.c", + "pngread.c", + "pngrio.c", + "pngrtran.c", + "pngrutil.c", + "pngset.c", + "pngtrans.c", + "pngwio.c", + "pngwrite.c", + "pngwtran.c", + "pngwutil.c", + ], + hdrs = [ + "png.h", + "pngconf.h", + ], + includes = ["."], + linkopts = ["-lm"], + visibility = ["//visibility:public"], + deps = ["@zlib_archive//:zlib"], +) diff --git a/oss-internship-2020/guetzli/external/zlib.BUILD b/oss-internship-2020/guetzli/external/zlib.BUILD new file mode 100644 index 0000000..edb77fd --- /dev/null +++ b/oss-internship-2020/guetzli/external/zlib.BUILD @@ -0,0 +1,36 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # BSD/MIT-like license (for zlib) + +cc_library( + name = "zlib", + srcs = [ + "adler32.c", + "compress.c", + "crc32.c", + "crc32.h", + "deflate.c", + "deflate.h", + "gzclose.c", + "gzguts.h", + "gzlib.c", + "gzread.c", + "gzwrite.c", + "infback.c", + "inffast.c", + "inffast.h", + "inffixed.h", + "inflate.c", + "inflate.h", + "inftrees.c", + "inftrees.h", + "trees.c", + "trees.h", + "uncompr.c", + "zconf.h", + "zutil.c", + "zutil.h", + ], + hdrs = ["zlib.h"], + includes = ["."], +) diff --git a/oss-internship-2020/guetzli/guetzli_entry_points.cc b/oss-internship-2020/guetzli/guetzli_entry_points.cc new file mode 100644 index 0000000..446150c --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_entry_points.cc @@ -0,0 +1,245 @@ +#include "guetzli/jpeg_data_reader.h" +#include "guetzli/quality.h" +#include "guetzli_entry_points.h" +#include "png.h" +#include "sandboxed_api/sandbox2/util/fileops.h" + +#include +#include +#include +#include +#include + +namespace { + +inline uint8_t BlendOnBlack(const uint8_t val, const uint8_t alpha) { + return (static_cast(val) * static_cast(alpha) + 128) / 255; +} + +template +void CopyMemoryToLenVal(const T* data, size_t size, + sapi::LenValStruct* out_data) { + free(out_data->data); // Not sure about this + out_data->size = size; + T* new_out = static_cast(malloc(size)); + memcpy(new_out, data, size); + out_data->data = new_out; +} + +} // namespace + +extern "C" bool ProcessJPEGString(const guetzli::Params* params, + int verbose, + sapi::LenValStruct* in_data, + sapi::LenValStruct* out_data) +{ + std::string in_data_temp(static_cast(in_data->data), + in_data->size); + + guetzli::ProcessStats stats; + if (verbose > 0) { + stats.debug_output_file = stderr; + } + + std::string temp_out = ""; + auto result = guetzli::Process(*params, &stats, in_data_temp, &temp_out); + + if (result) { + CopyMemoryToLenVal(temp_out.data(), temp_out.size(), out_data); + } + + return result; +} + +extern "C" bool ProcessRGBData(const guetzli::Params* params, + int verbose, + sapi::LenValStruct* rgb, + int w, int h, + sapi::LenValStruct* out_data) +{ + std::vector in_data_temp; + in_data_temp.reserve(rgb->size); + + auto* rgb_data = static_cast(rgb->data); + std::copy(rgb_data, rgb_data + rgb->size, std::back_inserter(in_data_temp)); + + guetzli::ProcessStats stats; + if (verbose > 0) { + stats.debug_output_file = stderr; + } + + std::string temp_out = ""; + auto result = + guetzli::Process(*params, &stats, in_data_temp, w, h, &temp_out); + + //TODO: Move shared part of the code to another function + if (result) { + CopyMemoryToLenVal(temp_out.data(), temp_out.size(), out_data); + } + + return result; +} + +extern "C" bool ReadPng(sapi::LenValStruct* in_data, + int* xsize, int* ysize, + sapi::LenValStruct* rgb_out) +{ + std::string data(static_cast(in_data->data), in_data->size); + std::vector rgb; + + png_structp png_ptr = + png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); + if (!png_ptr) { + return false; + } + + png_infop info_ptr = png_create_info_struct(png_ptr); + if (!info_ptr) { + png_destroy_read_struct(&png_ptr, nullptr, nullptr); + return false; + } + + if (setjmp(png_jmpbuf(png_ptr)) != 0) { + // Ok we are here because of the setjmp. + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + return false; + } + + std::istringstream memstream(data, std::ios::in | std::ios::binary); + png_set_read_fn(png_ptr, static_cast(&memstream), [](png_structp png_ptr, png_bytep outBytes, png_size_t byteCountToRead) { + std::istringstream& memstream = *static_cast(png_get_io_ptr(png_ptr)); + + memstream.read(reinterpret_cast(outBytes), byteCountToRead); + + if (memstream.eof()) png_error(png_ptr, "unexpected end of data"); + if (memstream.fail()) png_error(png_ptr, "read from memory error"); + }); + + // The png_transforms flags are as follows: + // packing == convert 1,2,4 bit images, + // strip == 16 -> 8 bits / channel, + // shift == use sBIT dynamics, and + // expand == palettes -> rgb, grayscale -> 8 bit images, tRNS -> alpha. + const unsigned int png_transforms = + PNG_TRANSFORM_PACKING | PNG_TRANSFORM_EXPAND | PNG_TRANSFORM_STRIP_16; + + png_read_png(png_ptr, info_ptr, png_transforms, nullptr); + + png_bytep* row_pointers = png_get_rows(png_ptr, info_ptr); + + *xsize = png_get_image_width(png_ptr, info_ptr); + *ysize = png_get_image_height(png_ptr, info_ptr); + rgb.resize(3 * (*xsize) * (*ysize)); + + const int components = png_get_channels(png_ptr, info_ptr); + switch (components) { + case 1: { + // GRAYSCALE + for (int y = 0; y < *ysize; ++y) { + const uint8_t* row_in = row_pointers[y]; + uint8_t* row_out = &(rgb)[3 * y * (*xsize)]; + for (int x = 0; x < *xsize; ++x) { + const uint8_t gray = row_in[x]; + row_out[3 * x + 0] = gray; + row_out[3 * x + 1] = gray; + row_out[3 * x + 2] = gray; + } + } + break; + } + case 2: { + // GRAYSCALE + ALPHA + for (int y = 0; y < *ysize; ++y) { + const uint8_t* row_in = row_pointers[y]; + uint8_t* row_out = &(rgb)[3 * y * (*xsize)]; + for (int x = 0; x < *xsize; ++x) { + const uint8_t gray = BlendOnBlack(row_in[2 * x], row_in[2 * x + 1]); + row_out[3 * x + 0] = gray; + row_out[3 * x + 1] = gray; + row_out[3 * x + 2] = gray; + } + } + break; + } + case 3: { + // RGB + for (int y = 0; y < *ysize; ++y) { + const uint8_t* row_in = row_pointers[y]; + uint8_t* row_out = &(rgb)[3 * y * (*xsize)]; + memcpy(row_out, row_in, 3 * (*xsize)); + } + break; + } + case 4: { + // RGBA + for (int y = 0; y < *ysize; ++y) { + const uint8_t* row_in = row_pointers[y]; + uint8_t* row_out = &(rgb)[3 * y * (*xsize)]; + for (int x = 0; x < *xsize; ++x) { + const uint8_t alpha = row_in[4 * x + 3]; + row_out[3 * x + 0] = BlendOnBlack(row_in[4 * x + 0], alpha); + row_out[3 * x + 1] = BlendOnBlack(row_in[4 * x + 1], alpha); + row_out[3 * x + 2] = BlendOnBlack(row_in[4 * x + 2], alpha); + } + } + break; + } + default: + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + return false; + } + png_destroy_read_struct(&png_ptr, &info_ptr, nullptr); + + CopyMemoryToLenVal(rgb.data(), rgb.size(), rgb_out); + + return true; +} + +extern "C" bool ReadJpegData(sapi::LenValStruct* in_data, + int mode, + int* xsize, int* ysize) +{ + std::string data(static_cast(in_data->data), in_data->size); + guetzli::JPEGData jpg; + + auto result = guetzli::ReadJpeg(data, + static_cast(mode), &jpg); + + if (result) { + *xsize = jpg.width; + *ysize = jpg.height; + } + + return result; +} + +extern "C" double ButteraugliScoreQuality(double quality) { + return guetzli::ButteraugliScoreForQuality(quality); +} + +extern "C" bool ReadDataFromFd(int fd, sapi::LenValStruct* out_data) { + struct stat file_data; + auto status = fstat(fd, &file_data); + + if (status < 0) { + return false; + } + + auto fsize = file_data.st_size; + + std::unique_ptr buf(new char[fsize]); + status = read(fd, buf.get(), fsize); + + if (status < 0) { + return false; + } + + CopyMemoryToLenVal(buf.get(), fsize, out_data); + + return true; +} + +extern "C" bool WriteDataToFd(int fd, sapi::LenValStruct* data) { + return sandbox2::file_util::fileops::WriteToFD(fd, + static_cast(data->data), data->size); +} \ No newline at end of file diff --git a/oss-internship-2020/guetzli/guetzli_entry_points.h b/oss-internship-2020/guetzli/guetzli_entry_points.h new file mode 100644 index 0000000..6fd0f11 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_entry_points.h @@ -0,0 +1,29 @@ +#pragma once + +#include "guetzli/processor.h" +#include "sandboxed_api/lenval_core.h" +#include "sandboxed_api/vars.h" + +extern "C" bool ProcessJPEGString(const guetzli::Params* params, + int verbose, + sapi::LenValStruct* in_data, + sapi::LenValStruct* out_data); + +extern "C" bool ProcessRGBData(const guetzli::Params* params, + int verbose, + sapi::LenValStruct* rgb, + int w, int h, + sapi::LenValStruct* out_data); + +extern "C" bool ReadPng(sapi::LenValStruct* in_data, + int* xsize, int* ysize, + sapi::LenValStruct* rgb_out); + +extern "C" bool ReadJpegData(sapi::LenValStruct* in_data, + int mode, int* xsize, int* ysize); + +extern "C" double ButteraugliScoreQuality(double quality); + +extern "C" bool ReadDataFromFd(int fd, sapi::LenValStruct* out_data); + +extern "C" bool WriteDataToFd(int fd, sapi::LenValStruct* data); \ No newline at end of file diff --git a/oss-internship-2020/guetzli/guetzli_sandbox.h b/oss-internship-2020/guetzli/guetzli_sandbox.h new file mode 100644 index 0000000..3fa2b10 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_sandbox.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include + +#include "guetzli_sapi.sapi.h" +#include "sandboxed_api/sandbox2/policy.h" +#include "sandboxed_api/sandbox2/policybuilder.h" +#include "sandboxed_api/util/flag.h" + +namespace guetzli { +namespace sandbox { + +class GuetzliSapiSandbox : public GuetzliSandbox { + public: + std::unique_ptr ModifyPolicy( + sandbox2::PolicyBuilder*) override { + + return sandbox2::PolicyBuilder() + .AllowStaticStartup() + .AllowRead() + .AllowSystemMalloc() + .AllowWrite() + .AllowExit() + .AllowStat() + .AllowSyscalls({ + __NR_futex, + __NR_close, + __NR_recvmsg // Seems like this one needed to work with remote file descriptors + }) + .BuildOrDie(); + } +}; + +} // namespace sandbox +} // namespace guetzli \ No newline at end of file diff --git a/oss-internship-2020/guetzli/guetzli_sandboxed.cc b/oss-internship-2020/guetzli/guetzli_sandboxed.cc new file mode 100644 index 0000000..a62f249 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_sandboxed.cc @@ -0,0 +1,160 @@ +#include +#include +#include +#include +#include + +#include + +#include "guetzli_transaction.h" +#include "sandboxed_api/sandbox2/util/fileops.h" +#include "sandboxed_api/util/statusor.h" + +namespace { + +constexpr int kDefaultJPEGQuality = 95; +constexpr int kDefaultMemlimitMB = 6000; // in MB +//constexpr absl::string_view kMktempSuffix = "XXXXXX"; + +// sapi::StatusOr> CreateNamedTempFile( +// absl::string_view prefix) { +// std::string name_template = absl::StrCat(prefix, kMktempSuffix); +// int fd = mkstemp(&name_template[0]); +// if (fd < 0) { +// return absl::UnknownError("Error creating temp file"); +// } +// return std::pair{std::move(name_template), fd}; +// } + +void TerminateHandler() { + fprintf(stderr, "Unhandled exception. Most likely insufficient memory available.\n" + "Make sure that there is 300MB/MPix of memory available.\n"); + exit(1); +} + +void Usage() { + fprintf(stderr, + "Guetzli JPEG compressor. Usage: \n" + "guetzli [flags] input_filename output_filename\n" + "\n" + "Flags:\n" + " --verbose - Print a verbose trace of all attempts to standard output.\n" + " --quality Q - Visual quality to aim for, expressed as a JPEG quality value.\n" + " Default value is %d.\n" + " --memlimit M - Memory limit in MB. Guetzli will fail if unable to stay under\n" + " the limit. Default limit is %d MB.\n" + " --nomemlimit - Do not limit memory usage.\n", kDefaultJPEGQuality, kDefaultMemlimitMB); + exit(1); +} + +} // namespace + +int main(int argc, const char** argv) { + std::set_terminate(TerminateHandler); + + int verbose = 0; + int quality = kDefaultJPEGQuality; + int memlimit_mb = kDefaultMemlimitMB; + + int opt_idx = 1; + for(;opt_idx < argc;opt_idx++) { + if (strnlen(argv[opt_idx], 2) < 2 || argv[opt_idx][0] != '-' || argv[opt_idx][1] != '-') + break; + + if (!strcmp(argv[opt_idx], "--verbose")) { + verbose = 1; + } else if (!strcmp(argv[opt_idx], "--quality")) { + opt_idx++; + if (opt_idx >= argc) + Usage(); + quality = atoi(argv[opt_idx]); + } else if (!strcmp(argv[opt_idx], "--memlimit")) { + opt_idx++; + if (opt_idx >= argc) + Usage(); + memlimit_mb = atoi(argv[opt_idx]); + } else if (!strcmp(argv[opt_idx], "--nomemlimit")) { + memlimit_mb = -1; + } else if (!strcmp(argv[opt_idx], "--")) { + opt_idx++; + break; + } else { + fprintf(stderr, "Unknown commandline flag: %s\n", argv[opt_idx]); + Usage(); + } + } + + if (argc - opt_idx != 2) { + Usage(); + } + + sandbox2::file_util::fileops::FDCloser in_fd_closer( + open(argv[opt_idx], O_RDONLY)); + + if (in_fd_closer.get() < 0) { + fprintf(stderr, "Can't open input file: %s\n", argv[opt_idx]); + return 1; + } + + // auto out_temp_file = CreateNamedTempFile("/tmp/" + std::string(argv[opt_idx + 1])); + // if (!out_temp_file.ok()) { + // fprintf(stderr, "Can't create temporary output file: %s\n", + // argv[opt_idx + 1]); + // return 1; + // } + // sandbox2::file_util::fileops::FDCloser out_fd_closer( + // out_temp_file.value().second); + + // if (unlink(out_temp_file.value().first.c_str()) < 0) { + // fprintf(stderr, "Error unlinking temp out file: %s\n", + // out_temp_file.value().first.c_str()); + // return 1; + // } + + sandbox2::file_util::fileops::FDCloser out_fd_closer( + open(".", O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR)); + + if (out_fd_closer.get() < 0) { + fprintf(stderr, "Can't create temporary output file: %s\n", argv[opt_idx]); + return 1; + } + + // sandbox2::file_util::fileops::FDCloser out_fd_closer(open(argv[opt_idx + 1], + // O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)); + + // if (out_fd_closer.get() < 0) { + // fprintf(stderr, "Can't open output file: %s\n", argv[opt_idx + 1]); + // return 1; + // } + + guetzli::sandbox::TransactionParams params = { + in_fd_closer.get(), + out_fd_closer.get(), + verbose, + quality, + memlimit_mb + }; + + guetzli::sandbox::GuetzliTransaction transaction(std::move(params)); + auto result = transaction.Run(); + + if (result.ok()) { + if (access(argv[opt_idx + 1], F_OK) != -1) { + if (remove(argv[opt_idx + 1]) < 0) { + fprintf(stderr, "Error deleting existing output file: %s\n", + argv[opt_idx + 1]); + } + } + + std::stringstream path; + path << "/proc/self/fd/" << out_fd_closer.get(); + linkat(AT_FDCWD, path.str().c_str(), AT_FDCWD, argv[opt_idx + 1], + AT_SYMLINK_FOLLOW); + } + else { + fprintf(stderr, "%s\n", result.ToString().c_str()); // Use cerr instead ? + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/oss-internship-2020/guetzli/guetzli_transaction.cc b/oss-internship-2020/guetzli/guetzli_transaction.cc new file mode 100644 index 0000000..9403783 --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_transaction.cc @@ -0,0 +1,160 @@ +#include "guetzli_transaction.h" + +#include + +namespace guetzli { +namespace sandbox { + +absl::Status GuetzliTransaction::Init() { + SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&in_fd_)); + SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&out_fd_)); + + if (in_fd_.GetRemoteFd() < 0) { + return absl::FailedPreconditionError( + "Error receiving remote FD: remote input fd is set to -1"); + } + if (out_fd_.GetRemoteFd() < 0) { + return absl::FailedPreconditionError( + "Error receiving remote FD: remote output fd is set to -1"); + } + + return absl::OkStatus(); +} + + absl::Status GuetzliTransaction::ProcessPng(GuetzliAPi* api, + sapi::v::Struct* params, + sapi::v::LenVal* input, + sapi::v::LenVal* output) const { + sapi::v::Int xsize; + sapi::v::Int ysize; + sapi::v::LenVal rgb_in(0); + + auto read_result = api->ReadPng(input->PtrBefore(), xsize.PtrBoth(), + ysize.PtrBoth(), rgb_in.PtrBoth()); + + if (!read_result.value_or(false)) { + return absl::FailedPreconditionError( + "Error reading PNG data from input file" + ); + } + + double pixels = static_cast(xsize.GetValue()) * ysize.GetValue(); + if (params_.memlimit_mb != -1 + && (pixels * kBytesPerPixel / (1 << 20) > params_.memlimit_mb + || params_.memlimit_mb < kLowestMemusageMB)) { + return absl::FailedPreconditionError( + "Memory limit would be exceeded" + ); + } + + auto result = api->ProcessRGBData(params->PtrBefore(), params_.verbose, + rgb_in.PtrBefore(), xsize.GetValue(), + ysize.GetValue(), output->PtrBoth()); + if (!result.value_or(false)) { + return absl::FailedPreconditionError( + "Guetzli processing failed" + ); + } + + return absl::OkStatus(); + } + + absl::Status GuetzliTransaction::ProcessJpeg(GuetzliApi* api, + sapi::v::Struct* params, + sapi::v::LenVal* input, + sapi::v::LenVal* output) const { + ::sapi::v::Int xsize; + ::sapi::v::Int ysize; + auto read_result = api->ReadJpegData(input->PtrBefore(), 0, xsize.PtrBoth(), + ysize.PtrBoth()); + + if (!read_result.value_or(false)) { + return absl::FailedPreconditionError( + "Error reading JPG data from input file" + ); + } + + double pixels = static_cast(xsize.GetValue()) * ysize.GetValue(); + if (params_.memlimit_mb != -1 + && (pixels * kBytesPerPixel / (1 << 20) > params_.memlimit_mb + || params_.memlimit_mb < kLowestMemusageMB)) { + return absl::FailedPreconditionError( + "Memory limit would be exceeded" + ); + } + + auto result = api->ProcessJPEGString(params->PtrBefore(), params_.verbose, + input->PtrBefore(), output->PtrBoth()); + + if (!result.value_or(false)) { + return absl::FailedPreconditionError( + "Guetzli processing failed" + ); + } + + return absl::OkStatus(); + } + +absl::Status GuetzliTransaction::Main() { + GuetzliApi api(sandbox()); + + sapi::v::LenVal input(0); + sapi::v::LenVal output(0); + sapi::v::Struct params; + + auto read_result = api.ReadDataFromFd(in_fd_.GetRemoteFd(), input.PtrBoth()); + + if (!read_result.value_or(false)) { + return absl::FailedPreconditionError( + "Error reading data inside sandbox" + ); + } + + auto score_quality_result = api.ButteraugliScoreQuality(params_.quality); + + if (!score_quality_result.ok()) { + return absl::FailedPreconditionError( + "Error calculating butteraugli score" + ); + } + + params.mutable_data()->butteraugli_target = score_quality_result.value(); + + static const unsigned char kPNGMagicBytes[] = { + 0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', + }; + + if (input.GetDataSize() >= 8 && + memcmp(input.GetData(), kPNGMagicBytes, sizeof(kPNGMagicBytes)) == 0) { + auto process_status = ProcessPng(&api, ¶ms, &input, &output); + + if (!process_status.ok()) { + return process_status; + } + } else { + auto process_status = ProcessJpeg(&api, ¶ms, &input, &output); + + if (!process_status.ok()) { + return process_status; + } + } + + auto write_result = api.WriteDataToFd(out_fd_.GetRemoteFd(), + output.PtrBefore()); + + if (!write_result.value_or(false)) { + return absl::FailedPreconditionError( + "Error writing file inside sandbox" + ); + } + + return absl::OkStatus(); +} + +time_t GuetzliTransaction::CalculateTimeLimitFromImageSize( + uint64_t pixels) const { + return (pixels / kMpixPixels + 5) * 60; +} + +} // namespace sandbox +} // namespace guetzli \ No newline at end of file diff --git a/oss-internship-2020/guetzli/guetzli_transaction.h b/oss-internship-2020/guetzli/guetzli_transaction.h new file mode 100644 index 0000000..5d6e83a --- /dev/null +++ b/oss-internship-2020/guetzli/guetzli_transaction.h @@ -0,0 +1,216 @@ +#pragma once + +#include +#include + +#include "guetzli_sandbox.h" +#include "sandboxed_api/transaction.h" +#include "sandboxed_api/vars.h" + +namespace guetzli { +namespace sandbox { + +constexpr int kDefaultTransactionRetryCount = 1; +constexpr uint64_t kMpixPixels = 1'000'000; + +constexpr int kBytesPerPixel = 350; +constexpr int kLowestMemusageMB = 100; // in MB + +struct TransactionParams { + int in_fd; + int out_fd; + int verbose; + int quality; + int memlimit_mb; +}; + +//Add optional time limit/retry count as a constructors arguments +//Use differenet status errors +class GuetzliTransaction : public sapi::Transaction { + public: + GuetzliTransaction(TransactionParams&& params) + : sapi::Transaction(std::make_unique()) + , params_(std::move(params)) + , in_fd_(params_.in_fd) + , out_fd_(params_.out_fd) + { + sapi::Transaction::set_retry_count(kDefaultTransactionRetryCount); + sapi::Transaction::SetTimeLimit(0); + } + + private: + absl::Status Init() override; + absl::Status Main() final; + + absl::Status ProcessPng(GuetzliApi* api, + sapi::v::Struct* params, + sapi::v::LenVal* input, + sapi::v::LenVal* output) const; + + absl::Status ProcessJpeg(GuetzliApi* api, + sapi::v::Struct* params, + sapi::v::LenVal* input, + sapi::v::LenVal* output) const; + + // As guetzli takes roughly 1 minute of CPU per 1 MPix we need to calculate + // approximate time for transaction to complete + time_t CalculateTimeLimitFromImageSize(uint64_t pixels) const; + + const TransactionParams params_; + sapi::v::Fd in_fd_; + sapi::v::Fd out_fd_; +}; + +absl::Status GuetzliTransaction::Init() { + SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&in_fd_)); + SAPI_RETURN_IF_ERROR(sandbox()->TransferToSandboxee(&out_fd_)); + + if (in_fd_.GetRemoteFd() < 0) { + return absl::FailedPreconditionError( + "Error receiving remote FD: remote input fd is set to -1"); + } + if (out_fd_.GetRemoteFd() < 0) { + return absl::FailedPreconditionError( + "Error receiving remote FD: remote output fd is set to -1"); + } + + return absl::OkStatus(); +} + + absl::Status GuetzliTransaction::ProcessPng(GuetzliApi* api, + sapi::v::Struct* params, + sapi::v::LenVal* input, + sapi::v::LenVal* output) const { + sapi::v::Int xsize; + sapi::v::Int ysize; + sapi::v::LenVal rgb_in(0); + + auto read_result = api->ReadPng(input->PtrBefore(), xsize.PtrBoth(), + ysize.PtrBoth(), rgb_in.PtrBoth()); + + if (!read_result.value_or(false)) { + return absl::FailedPreconditionError( + "Error reading PNG data from input file" + ); + } + + double pixels = static_cast(xsize.GetValue()) * ysize.GetValue(); + if (params_.memlimit_mb != -1 + && (pixels * kBytesPerPixel / (1 << 20) > params_.memlimit_mb + || params_.memlimit_mb < kLowestMemusageMB)) { + return absl::FailedPreconditionError( + "Memory limit would be exceeded" + ); + } + + auto result = api->ProcessRGBData(params->PtrBefore(), params_.verbose, + rgb_in.PtrBefore(), xsize.GetValue(), + ysize.GetValue(), output->PtrBoth()); + if (!result.value_or(false)) { + return absl::FailedPreconditionError( + "Guetzli processing failed" + ); + } + + return absl::OkStatus(); + } + + absl::Status GuetzliTransaction::ProcessJpeg(GuetzliApi* api, + sapi::v::Struct* params, + sapi::v::LenVal* input, + sapi::v::LenVal* output) const { + sapi::v::Int xsize; + sapi::v::Int ysize; + auto read_result = api->ReadJpegData(input->PtrBefore(), 0, xsize.PtrBoth(), + ysize.PtrBoth()); + + if (!read_result.value_or(false)) { + return absl::FailedPreconditionError( + "Error reading JPG data from input file" + ); + } + + double pixels = static_cast(xsize.GetValue()) * ysize.GetValue(); + if (params_.memlimit_mb != -1 + && (pixels * kBytesPerPixel / (1 << 20) > params_.memlimit_mb + || params_.memlimit_mb < kLowestMemusageMB)) { + return absl::FailedPreconditionError( + "Memory limit would be exceeded" + ); + } + + auto result = api->ProcessJPEGString(params->PtrBefore(), params_.verbose, + input->PtrBefore(), output->PtrBoth()); + + if (!result.value_or(false)) { + return absl::FailedPreconditionError( + "Guetzli processing failed" + ); + } + + return absl::OkStatus(); + } + +absl::Status GuetzliTransaction::Main() { + GuetzliApi api(sandbox()); + + sapi::v::LenVal input(0); + sapi::v::LenVal output(0); + sapi::v::Struct params; + + auto read_result = api.ReadDataFromFd(in_fd_.GetRemoteFd(), input.PtrBoth()); + + if (!read_result.value_or(false)) { + return absl::FailedPreconditionError( + "Error reading data inside sandbox" + ); + } + + auto score_quality_result = api.ButteraugliScoreQuality(params_.quality); + + if (!score_quality_result.ok()) { + return absl::FailedPreconditionError( + "Error calculating butteraugli score" + ); + } + + params.mutable_data()->butteraugli_target = score_quality_result.value(); + + static const unsigned char kPNGMagicBytes[] = { + 0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n', + }; + + if (input.GetDataSize() >= 8 && + memcmp(input.GetData(), kPNGMagicBytes, sizeof(kPNGMagicBytes)) == 0) { + auto process_status = ProcessPng(&api, ¶ms, &input, &output); + + if (!process_status.ok()) { + return process_status; + } + } else { + auto process_status = ProcessJpeg(&api, ¶ms, &input, &output); + + if (!process_status.ok()) { + return process_status; + } + } + + auto write_result = api.WriteDataToFd(out_fd_.GetRemoteFd(), + output.PtrBefore()); + + if (!write_result.value_or(false)) { + return absl::FailedPreconditionError( + "Error writing file inside sandbox" + ); + } + + return absl::OkStatus(); +} + +time_t GuetzliTransaction::CalculateTimeLimitFromImageSize( + uint64_t pixels) const { + return (pixels / kMpixPixels + 5) * 60; +} + +} // namespace sandbox +} // namespace guetzli diff --git a/oss-internship-2020/guetzli/tests/BUILD.bazel b/oss-internship-2020/guetzli/tests/BUILD.bazel new file mode 100644 index 0000000..b7c7517 --- /dev/null +++ b/oss-internship-2020/guetzli/tests/BUILD.bazel @@ -0,0 +1,22 @@ +# cc_test( +# name = "transaction_tests", +# srcs = ["guetzli_transaction_test.cc"], +# visibility=["//visibility:public"], +# includes = ["."], +# deps = [ +# "//:guetzli_sapi", +# "@googletest//:gtest_main" +# ], +# ) + +# cc_test( +# name = "sapi_lib_tests", +# srcs = ["guetzli_sapi_test.cc"], +# visibility=["//visibility:public"], +# includes=[".."], +# deps = [ +# "//:guetzli_sapi", +# "@googletest//:gtest_main" +# ], +# data = glob(["testdata/*"]) +# ) \ No newline at end of file diff --git a/oss-internship-2020/guetzli/tests/guetzli_sapi_test.cc b/oss-internship-2020/guetzli/tests/guetzli_sapi_test.cc new file mode 100644 index 0000000..73e54d6 --- /dev/null +++ b/oss-internship-2020/guetzli/tests/guetzli_sapi_test.cc @@ -0,0 +1,165 @@ +#include "gtest/gtest.h" +#include "guetzli_sandbox.h" +#include "sandboxed_api/sandbox2/util/fileops.h" +#include "sandboxed_api/vars.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace guetzli { +namespace sandbox { +namespace tests { + +namespace { + +constexpr const char* IN_PNG_FILENAME = "bees.png"; +constexpr const char* IN_JPG_FILENAME = "landscape.jpg"; + +constexpr int IN_PNG_FILE_SIZE = 177'424; +constexpr int IN_JPG_FILE_SIZE = 14'418; + +constexpr int DEFAULT_QUALITY_TARGET = 95; + +constexpr const char* RELATIVE_PATH_TO_TESTDATA = + "/guetzli/guetzli-sandboxed/tests/testdata/"; + +std::string GetPathToInputFile(const char* filename) { + return std::string(getenv("TEST_SRCDIR")) + + std::string(RELATIVE_PATH_TO_TESTDATA) + + std::string(filename); +} + +std::string ReadFromFile(const std::string& filename) { + std::ifstream stream(filename, std::ios::binary); + + if (!stream.is_open()) { + return ""; + } + + std::stringstream result; + result << stream.rdbuf(); + return result.str(); +} + +} // namespace + +class GuetzliSapiTest : public ::testing::Test { +protected: + void SetUp() override { + sandbox_ = std::make_unique(); + sandbox_->Init().IgnoreError(); + api_ = std::make_unique(sandbox_.get()); + } + + std::unique_ptr sandbox_; + std::unique_ptr api_; +}; + +TEST_F(GuetzliSapiTest, ReadDataFromFd) { + std::string input_file_path = GetPathToInputFile(IN_PNG_FILENAME); + int fd = open(input_file_path.c_str(), O_RDONLY); + ASSERT_TRUE(fd != -1) << "Error opening input file"; + sapi::v::Fd remote_fd(fd); + auto send_fd_status = sandbox_->TransferToSandboxee(&remote_fd); + ASSERT_TRUE(send_fd_status.ok()) << "Error sending fd to sandboxee"; + ASSERT_TRUE(remote_fd.GetRemoteFd() != -1) << "Error opening remote fd"; + sapi::v::LenVal data(0); + auto read_status = + api_->ReadDataFromFd(remote_fd.GetRemoteFd(), data.PtrBoth()); + ASSERT_TRUE(read_status.value_or(false)) << "Error reading data from fd"; + ASSERT_EQ(data.GetDataSize(), IN_PNG_FILE_SIZE) << "Wrong size of file"; +} + +// TEST_F(GuetzliSapiTest, WriteDataToFd) { + +// } + +TEST_F(GuetzliSapiTest, ReadPng) { + std::string data = ReadFromFile(GetPathToInputFile(IN_PNG_FILENAME)); + ASSERT_EQ(data.size(), IN_PNG_FILE_SIZE) << "Error reading input file"; + sapi::v::LenVal in_data(data.data(), data.size()); + sapi::v::Int xsize, ysize; + sapi::v::LenVal rgb_out(0); + + auto status = api_->ReadPng(in_data.PtrBefore(), xsize.PtrBoth(), + ysize.PtrBoth(), rgb_out.PtrBoth()); + ASSERT_TRUE(status.value_or(false)) << "Error processing png data"; + ASSERT_EQ(xsize.GetValue(), 444) << "Error parsing width"; + ASSERT_EQ(ysize.GetValue(), 258) << "Error parsing height"; +} + +TEST_F(GuetzliSapiTest, ReadJpeg) { + std::string data = ReadFromFile(GetPathToInputFile(IN_JPG_FILENAME)); + ASSERT_EQ(data.size(), IN_JPG_FILE_SIZE) << "Error reading input file"; + sapi::v::LenVal in_data(data.data(), data.size()); + sapi::v::Int xsize, ysize; + + auto status = api_->ReadJpegData(in_data.PtrBefore(), 0, + xsize.PtrBoth(), ysize.PtrBoth()); + ASSERT_TRUE(status.value_or(false)) << "Error processing jpeg data"; + ASSERT_EQ(xsize.GetValue(), 180) << "Error parsing width"; + ASSERT_EQ(ysize.GetValue(), 180) << "Error parsing height"; +} + +// This test can take up to few minutes depending on your hardware +TEST_F(GuetzliSapiTest, ProcessRGB) { + std::string data = ReadFromFile(GetPathToInputFile(IN_PNG_FILENAME)); + ASSERT_EQ(data.size(), IN_PNG_FILE_SIZE) << "Error reading input file"; + sapi::v::LenVal in_data(data.data(), data.size()); + sapi::v::Int xsize, ysize; + sapi::v::LenVal rgb_out(0); + + auto status = api_->ReadPng(in_data.PtrBefore(), xsize.PtrBoth(), + ysize.PtrBoth(), rgb_out.PtrBoth()); + ASSERT_TRUE(status.value_or(false)) << "Error processing png data"; + ASSERT_EQ(xsize.GetValue(), 444) << "Error parsing width"; + ASSERT_EQ(ysize.GetValue(), 258) << "Error parsing height"; + auto quality = + api_->ButteraugliScoreQuality(static_cast(DEFAULT_QUALITY_TARGET)); + ASSERT_TRUE(quality.ok()) << "Error calculating butteraugli quality"; + sapi::v::Struct params; + sapi::v::LenVal out_data(0); + params.mutable_data()->butteraugli_target = quality.value(); + + status = api_->ProcessRGBData(params.PtrBefore(), 0, rgb_out.PtrBefore(), + xsize.GetValue(), ysize.GetValue(), out_data.PtrBoth()); + ASSERT_TRUE(status.value_or(false)) << "Error processing png file"; + ASSERT_EQ(out_data.GetDataSize(), 38'625); + //ADD COMPARSION WITH REFERENCE OUTPUT +} + +// This test can take up to few minutes depending on your hardware +TEST_F(GuetzliSapiTest, ProcessJpeg) { + std::string data = ReadFromFile(GetPathToInputFile(IN_JPG_FILENAME)); + ASSERT_EQ(data.size(), IN_JPG_FILE_SIZE) << "Error reading input file"; + sapi::v::LenVal in_data(data.data(), data.size()); + sapi::v::Int xsize, ysize; + + auto status = api_->ReadJpegData(in_data.PtrBefore(), 0, + xsize.PtrBoth(), ysize.PtrBoth()); + ASSERT_TRUE(status.value_or(false)) << "Error processing jpeg data"; + ASSERT_EQ(xsize.GetValue(), 180) << "Error parsing width"; + ASSERT_EQ(ysize.GetValue(), 180) << "Error parsing height"; + + auto quality = + api_->ButteraugliScoreQuality(static_cast(DEFAULT_QUALITY_TARGET)); + ASSERT_TRUE(quality.ok()) << "Error calculating butteraugli quality"; + sapi::v::Struct params; + params.mutable_data()->butteraugli_target = quality.value(); + sapi::v::LenVal out_data(0); + + status = api_->ProcessJPEGString(params.PtrBefore(), 0, in_data.PtrBefore(), + out_data.PtrBoth()); + ASSERT_TRUE(status.value_or(false)) << "Error processing jpeg file"; + ASSERT_EQ(out_data.GetDataSize(), 10'816); + //ADD COMPARSION WITH REFERENCE OUTPUT +} + +} // namespace tests +} // namespace sandbox +} // namespace guetzli \ No newline at end of file diff --git a/oss-internship-2020/guetzli/tests/guetzli_transaction_test.cc b/oss-internship-2020/guetzli/tests/guetzli_transaction_test.cc new file mode 100644 index 0000000..8471d1e --- /dev/null +++ b/oss-internship-2020/guetzli/tests/guetzli_transaction_test.cc @@ -0,0 +1,2 @@ +#include "gtest/gtest.h" +#include "guetzli_transaction.h" diff --git a/oss-internship-2020/guetzli/tests/testdata/bees.png b/oss-internship-2020/guetzli/tests/testdata/bees.png new file mode 100644 index 0000000..11640c7 Binary files /dev/null and b/oss-internship-2020/guetzli/tests/testdata/bees.png differ diff --git a/oss-internship-2020/guetzli/tests/testdata/landscape.jpg b/oss-internship-2020/guetzli/tests/testdata/landscape.jpg new file mode 100644 index 0000000..5c87177 Binary files /dev/null and b/oss-internship-2020/guetzli/tests/testdata/landscape.jpg differ