zopfli: introduce a wrapper

The goal is to use a file descriptor as an input.
Thanks to that we shouldn't send a chunk of memory over
expensive protocol.

contrib/zopfli/CMakeLists.txt

@@ -32,6 +32,8 @@ FetchContent_Declare(zopfli
 )
 FetchContent_MakeAvailable(zopfli)
 
+add_subdirectory(wrapper)
+
 add_sapi_library(
   sapi_zopfli
 
@@ -44,12 +46,14 @@ add_sapi_library(
     ZopfliZlibCompress
     ZopfliGzipCompress
 
+    ZopfliCompressFD
   INPUTS
     ${zopfli_SOURCE_DIR}/src/zopfli/deflate.h
     ${zopfli_SOURCE_DIR}/src/zopfli/gzip_container.h
     ${zopfli_SOURCE_DIR}/src/zopfli/zlib_container.h
+    wrapper/wrapper_zopfli.h
 
-  LIBRARY Zopfli::libzopfli
+  LIBRARY wrapper_zopfli
   LIBRARY_NAME Zopfli
   NAMESPACE ""
 )

contrib/zopfli/example/main.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <fcntl.h>
 #include <unistd.h>
 
 #include <fstream>
@@ -23,9 +24,39 @@
 #include "contrib/zopfli/sandboxed.h"
 #include "contrib/zopfli/utils/utils_zopfli.h"
 
+ABSL_FLAG(bool, stream, false, "stream memory to sandbox");
 ABSL_FLAG(bool, zlib, false, "zlib compression");
 ABSL_FLAG(bool, gzip, false, "gzip compression");
 
+absl::Status CompressMain(ZopfliApi& api, std::string& infile_s,
+                          std::string& outfile_s, ZopfliFormat format) {
+  std::ifstream infile(infile_s, std::ios::binary);
+  if (!infile.is_open()) {
+    return absl::UnavailableError(absl::StrCat("Unable to open ", infile_s));
+  }
+  std::ofstream outfile(outfile_s, std::ios::binary);
+  if (!outfile.is_open()) {
+    return absl::UnavailableError(absl::StrCat("Unable to open ", outfile_s));
+  }
+
+  return Compress(api, infile, outfile, format);
+}
+
+absl::Status CompressMainFD(ZopfliApi& api, std::string& infile_s,
+                            std::string& outfile_s, ZopfliFormat format) {
+  sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY));
+  if (infd.GetValue() < 0) {
+    return absl::UnavailableError(absl::StrCat("Unable to open ", infile_s));
+  }
+
+  sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY | O_CREAT));
+  if (outfd.GetValue() < 0) {
+    return absl::UnavailableError(absl::StrCat("Unable to open ", outfile_s));
+  }
+
+  return (CompressFD(api, infd, outfd, format));
+}
+
 int main(int argc, char* argv[]) {
   std::string prog_name(argv[0]);
   google::InitGoogleLogging(argv[0]);
@@ -36,22 +67,13 @@ int main(int argc, char* argv[]) {
     return EXIT_FAILURE;
   }
 
-  std::ifstream infile(args[1], std::ios::binary);
-  if (!infile.is_open()) {
-    std::cerr << "Unable to open " << args[1] << std::endl;
-    return EXIT_FAILURE;
-  }
-  std::ofstream outfile(args[2], std::ios::binary);
-  if (!outfile.is_open()) {
-    std::cerr << "Unable to open " << args[2] << std::endl;
-    return EXIT_FAILURE;
-  }
-
   ZopfliSapiSandbox sandbox;
   if (!sandbox.Init().ok()) {
     std::cerr << "Unable to start sandbox\n";
     return EXIT_FAILURE;
   }
+  std::string infile_s(args[1]);
+  std::string outfile_s(args[2]);
 
   ZopfliApi api(&sandbox);
 
@@ -62,7 +84,13 @@ int main(int argc, char* argv[]) {
     format = ZOPFLI_FORMAT_GZIP;
   }
 
-  absl::Status status = Compress(api, infile, outfile, format);
+  absl::Status status;
+  if (absl::GetFlag(FLAGS_stream)) {
+    status = CompressMain(api, infile_s, outfile_s, format);
+  } else {
+    status = CompressMainFD(api, infile_s, outfile_s, format);
+  }
+
   if (!status.ok()) {
     std::cerr << "Unable to compress file.\n";
     std::cerr << status << std::endl;

contrib/zopfli/sandboxed.h

@@ -28,12 +28,13 @@ class ZopfliSapiSandbox : public ZopfliSandbox {
   std::unique_ptr<sandbox2::Policy> ModifyPolicy(
       sandbox2::PolicyBuilder *) override {
     return sandbox2::PolicyBuilder()
-        .AllowStaticStartup()
+        .AllowDynamicStartup()
         .AllowWrite()
         .AllowExit()
         .AllowMmap()
         .AllowSystemMalloc()
         .AllowSyscalls({
+            __NR_recvmsg,
             __NR_sysinfo,
         })
 #ifdef __NR_open

contrib/zopfli/test/zopfli_test.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <fcntl.h>
+
 #include <fstream>
 
 #include "contrib/zopfli/sandboxed.h"
@@ -41,8 +43,9 @@ std::string GetTemporaryFile(const std::string& filename) {
 }
 
 class TestText : public testing::TestWithParam<ZopfliFormat> {};
-
 class TestBinary : public testing::TestWithParam<ZopfliFormat> {};
+class TestTextFD : public testing::TestWithParam<ZopfliFormat> {};
+class TestBinaryFD : public testing::TestWithParam<ZopfliFormat> {};
 
 TEST_P(TestText, Compress) {
   ZopfliSapiSandbox sandbox;
@@ -95,4 +98,68 @@ INSTANTIATE_TEST_SUITE_P(SandboxTest, TestBinary,
                          testing::Values(ZOPFLI_FORMAT_DEFLATE,
                                          ZOPFLI_FORMAT_GZIP,
                                          ZOPFLI_FORMAT_ZLIB));
+
+TEST_P(TestTextFD, Compress) {
+  ZopfliSapiSandbox sandbox;
+  ASSERT_THAT(sandbox.Init(), IsOk()) << "Couldn't initialize Sandboxed API";
+  ZopfliApi api = ZopfliApi(&sandbox);
+
+  std::string infile_s = GetTestFilePath("text");
+  std::string outfile_s = GetTemporaryFile("text.out");
+  ASSERT_THAT(outfile_s, Not(IsEmpty()));
+
+  sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY));
+  ASSERT_GE(infd.GetValue(), 0);
+
+  sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY));
+  ASSERT_GE(outfd.GetValue(), 0);
+
+  absl::Status status = CompressFD(api, infd, outfd, GetParam());
+  ASSERT_THAT(status, IsOk()) << "Unable to compress file";
+
+  off_t inpos = lseek(infd.GetValue(), 0, SEEK_END);
+  EXPECT_GE(inpos, 0);
+
+  off_t outpos = lseek(outfd.GetValue(), 0, SEEK_END);
+  EXPECT_GE(outpos, 0);
+
+  EXPECT_LT(outpos, inpos);
+}
+
+INSTANTIATE_TEST_SUITE_P(SandboxTest, TestTextFD,
+                         testing::Values(ZOPFLI_FORMAT_DEFLATE,
+                                         ZOPFLI_FORMAT_GZIP,
+                                         ZOPFLI_FORMAT_ZLIB));
+
+TEST_P(TestBinaryFD, Compress) {
+  ZopfliSapiSandbox sandbox;
+  ASSERT_THAT(sandbox.Init(), IsOk()) << "Couldn't initialize Sandboxed API";
+  ZopfliApi api = ZopfliApi(&sandbox);
+
+  std::string infile_s = GetTestFilePath("binary");
+  std::string outfile_s = GetTemporaryFile("binary.out");
+  ASSERT_THAT(outfile_s, Not(IsEmpty()));
+
+  sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY));
+  ASSERT_GE(infd.GetValue(), 0);
+
+  sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY));
+  ASSERT_GE(outfd.GetValue(), 0);
+
+  absl::Status status = CompressFD(api, infd, outfd, GetParam());
+  ASSERT_THAT(status, IsOk()) << "Unable to compress file";
+
+  off_t inpos = lseek(infd.GetValue(), 0, SEEK_END);
+  EXPECT_GE(inpos, 0);
+
+  off_t outpos = lseek(outfd.GetValue(), 0, SEEK_END);
+  EXPECT_GE(outpos, 0);
+
+  EXPECT_LT(outpos, inpos);
+}
+
+INSTANTIATE_TEST_SUITE_P(SandboxTest, TestBinaryFD,
+                         testing::Values(ZOPFLI_FORMAT_DEFLATE,
+                                         ZOPFLI_FORMAT_GZIP,
+                                         ZOPFLI_FORMAT_ZLIB));
 }  // namespace

contrib/zopfli/utils/utils_zopfli.cc

@@ -54,3 +54,25 @@ absl::Status Compress(ZopfliApi& api, std::ifstream& instream,
   }
   return absl::OkStatus();
 }
+
+absl::Status CompressFD(ZopfliApi& api, sapi::v::Fd& infd, sapi::v::Fd& outfd,
+                        ZopfliFormat format) {
+  SAPI_RETURN_IF_ERROR(api.GetSandbox()->TransferToSandboxee(&infd));
+  SAPI_RETURN_IF_ERROR(api.GetSandbox()->TransferToSandboxee(&outfd));
+
+  sapi::v::Struct<ZopfliOptions> options;
+  SAPI_RETURN_IF_ERROR(api.ZopfliInitOptions(options.PtrAfter()));
+
+  SAPI_ASSIGN_OR_RETURN(
+      int ret, api.ZopfliCompressFD(options.PtrBefore(), format,
+                                    infd.GetRemoteFd(), outfd.GetRemoteFd()));
+
+  infd.CloseRemoteFd(api.GetSandbox()->rpc_channel()).IgnoreError();
+  outfd.CloseRemoteFd(api.GetSandbox()->rpc_channel()).IgnoreError();
+
+  if (ret == -1) {
+    return absl::UnavailableError("Unable to compress file");
+  }
+
+  return absl::OkStatus();
+}

contrib/zopfli/utils/utils_zopfli.h

@@ -23,4 +23,7 @@
 absl::Status Compress(ZopfliApi& api, std::ifstream& instream,
                       std::ofstream& outstream, ZopfliFormat format);
 
+absl::Status CompressFD(ZopfliApi& api, sapi::v::Fd& infd, sapi::v::Fd& outfd,
+                        ZopfliFormat format);
+
 #endif  // CONTRIB_ZOPFLI_UTILS_UTILS_ZOPFLI_H_

contrib/zopfli/wrapper/CMakeLists.txt

@@ -0,0 +1,28 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_library(
+  wrapper_zopfli STATIC
+
+  wrapper_zopfli.cc
+)
+
+target_link_libraries(wrapper_zopfli PUBLIC
+  Zopfli::libzopfli
+)
+
+target_include_directories(wrapper_zopfli PUBLIC
+  ${SAPI_SOURCE_DIR}
+  ${libzopfli_SOURCE_DIR}/src/zopfli
+)

contrib/zopfli/wrapper/wrapper_zopfli.cc

@@ -0,0 +1,50 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "contrib/zopfli/wrapper/wrapper_zopfli.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <cstdio>
+#include <cstdlib>
+#include <memory>
+
+int ZopfliCompressFD(const ZopfliOptions* options, ZopfliFormat output_type,
+                     int infd, int outfd) {
+  off_t insize = lseek(infd, 0, SEEK_END);
+  if (insize < 0) {
+    return -1;
+  }
+  if (lseek(infd, 0, SEEK_SET) < 0) {
+    return -1;
+  }
+
+  auto inbuf = std::make_unique<uint8_t[]>(insize);
+  if (read(infd, inbuf.get(), insize) != insize) {
+    return -1;
+  }
+
+  size_t outsize = 0;
+  uint8_t* outbuf = nullptr;
+  ZopfliCompress(options, output_type, inbuf.get(), insize, &outbuf, &outsize);
+  size_t retsize = write(outfd, outbuf, outsize);
+  free(outbuf);
+
+  if (outsize != retsize) {
+    return -1;
+  }
+
+  return 0;
+}

contrib/zopfli/wrapper/wrapper_zopfli.h

@@ -0,0 +1,25 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_
+#define CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_
+
+#include "zopfli.h"
+
+extern "C" {
+int ZopfliCompressFD(const ZopfliOptions* options, ZopfliFormat output_type,
+                     int infd, int outfd);
+};
+
+#endif  // CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_