From b3aca9ef4860a3d85933777a47a60f06a154ae52 Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Sun, 6 Feb 2022 14:31:19 -0500 Subject: [PATCH] zopfli: introduce a wrapper The goal is to use a file descriptor as an input. Thanks to that we shouldn't send a chunk of memory over expensive protocol. --- contrib/zopfli/CMakeLists.txt | 6 ++- contrib/zopfli/example/main.cc | 52 +++++++++++++----- contrib/zopfli/sandboxed.h | 3 +- contrib/zopfli/test/zopfli_test.cc | 69 +++++++++++++++++++++++- contrib/zopfli/utils/utils_zopfli.cc | 22 ++++++++ contrib/zopfli/utils/utils_zopfli.h | 3 ++ contrib/zopfli/wrapper/CMakeLists.txt | 28 ++++++++++ contrib/zopfli/wrapper/wrapper_zopfli.cc | 50 +++++++++++++++++ contrib/zopfli/wrapper/wrapper_zopfli.h | 25 +++++++++ 9 files changed, 243 insertions(+), 15 deletions(-) create mode 100644 contrib/zopfli/wrapper/CMakeLists.txt create mode 100644 contrib/zopfli/wrapper/wrapper_zopfli.cc create mode 100644 contrib/zopfli/wrapper/wrapper_zopfli.h diff --git a/contrib/zopfli/CMakeLists.txt b/contrib/zopfli/CMakeLists.txt index 4de454d..995ffc1 100644 --- a/contrib/zopfli/CMakeLists.txt +++ b/contrib/zopfli/CMakeLists.txt @@ -32,6 +32,8 @@ FetchContent_Declare(zopfli ) FetchContent_MakeAvailable(zopfli) +add_subdirectory(wrapper) + add_sapi_library( sapi_zopfli @@ -44,12 +46,14 @@ add_sapi_library( ZopfliZlibCompress ZopfliGzipCompress + ZopfliCompressFD INPUTS ${zopfli_SOURCE_DIR}/src/zopfli/deflate.h ${zopfli_SOURCE_DIR}/src/zopfli/gzip_container.h ${zopfli_SOURCE_DIR}/src/zopfli/zlib_container.h + wrapper/wrapper_zopfli.h - LIBRARY Zopfli::libzopfli + LIBRARY wrapper_zopfli LIBRARY_NAME Zopfli NAMESPACE "" ) diff --git a/contrib/zopfli/example/main.cc b/contrib/zopfli/example/main.cc index 81b29ab..12469b8 100644 --- a/contrib/zopfli/example/main.cc +++ b/contrib/zopfli/example/main.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include @@ -23,9 +24,39 @@ #include "contrib/zopfli/sandboxed.h" #include "contrib/zopfli/utils/utils_zopfli.h" +ABSL_FLAG(bool, stream, false, "stream memory to sandbox"); ABSL_FLAG(bool, zlib, false, "zlib compression"); ABSL_FLAG(bool, gzip, false, "gzip compression"); +absl::Status CompressMain(ZopfliApi& api, std::string& infile_s, + std::string& outfile_s, ZopfliFormat format) { + std::ifstream infile(infile_s, std::ios::binary); + if (!infile.is_open()) { + return absl::UnavailableError(absl::StrCat("Unable to open ", infile_s)); + } + std::ofstream outfile(outfile_s, std::ios::binary); + if (!outfile.is_open()) { + return absl::UnavailableError(absl::StrCat("Unable to open ", outfile_s)); + } + + return Compress(api, infile, outfile, format); +} + +absl::Status CompressMainFD(ZopfliApi& api, std::string& infile_s, + std::string& outfile_s, ZopfliFormat format) { + sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY)); + if (infd.GetValue() < 0) { + return absl::UnavailableError(absl::StrCat("Unable to open ", infile_s)); + } + + sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY | O_CREAT)); + if (outfd.GetValue() < 0) { + return absl::UnavailableError(absl::StrCat("Unable to open ", outfile_s)); + } + + return (CompressFD(api, infd, outfd, format)); +} + int main(int argc, char* argv[]) { std::string prog_name(argv[0]); google::InitGoogleLogging(argv[0]); @@ -36,22 +67,13 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } - std::ifstream infile(args[1], std::ios::binary); - if (!infile.is_open()) { - std::cerr << "Unable to open " << args[1] << std::endl; - return EXIT_FAILURE; - } - std::ofstream outfile(args[2], std::ios::binary); - if (!outfile.is_open()) { - std::cerr << "Unable to open " << args[2] << std::endl; - return EXIT_FAILURE; - } - ZopfliSapiSandbox sandbox; if (!sandbox.Init().ok()) { std::cerr << "Unable to start sandbox\n"; return EXIT_FAILURE; } + std::string infile_s(args[1]); + std::string outfile_s(args[2]); ZopfliApi api(&sandbox); @@ -62,7 +84,13 @@ int main(int argc, char* argv[]) { format = ZOPFLI_FORMAT_GZIP; } - absl::Status status = Compress(api, infile, outfile, format); + absl::Status status; + if (absl::GetFlag(FLAGS_stream)) { + status = CompressMain(api, infile_s, outfile_s, format); + } else { + status = CompressMainFD(api, infile_s, outfile_s, format); + } + if (!status.ok()) { std::cerr << "Unable to compress file.\n"; std::cerr << status << std::endl; diff --git a/contrib/zopfli/sandboxed.h b/contrib/zopfli/sandboxed.h index e74d3ec..80442e3 100644 --- a/contrib/zopfli/sandboxed.h +++ b/contrib/zopfli/sandboxed.h @@ -28,12 +28,13 @@ class ZopfliSapiSandbox : public ZopfliSandbox { std::unique_ptr ModifyPolicy( sandbox2::PolicyBuilder *) override { return sandbox2::PolicyBuilder() - .AllowStaticStartup() + .AllowDynamicStartup() .AllowWrite() .AllowExit() .AllowMmap() .AllowSystemMalloc() .AllowSyscalls({ + __NR_recvmsg, __NR_sysinfo, }) #ifdef __NR_open diff --git a/contrib/zopfli/test/zopfli_test.cc b/contrib/zopfli/test/zopfli_test.cc index 0c060ff..2986cec 100644 --- a/contrib/zopfli/test/zopfli_test.cc +++ b/contrib/zopfli/test/zopfli_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include #include "contrib/zopfli/sandboxed.h" @@ -41,8 +43,9 @@ std::string GetTemporaryFile(const std::string& filename) { } class TestText : public testing::TestWithParam {}; - class TestBinary : public testing::TestWithParam {}; +class TestTextFD : public testing::TestWithParam {}; +class TestBinaryFD : public testing::TestWithParam {}; TEST_P(TestText, Compress) { ZopfliSapiSandbox sandbox; @@ -95,4 +98,68 @@ INSTANTIATE_TEST_SUITE_P(SandboxTest, TestBinary, testing::Values(ZOPFLI_FORMAT_DEFLATE, ZOPFLI_FORMAT_GZIP, ZOPFLI_FORMAT_ZLIB)); + +TEST_P(TestTextFD, Compress) { + ZopfliSapiSandbox sandbox; + ASSERT_THAT(sandbox.Init(), IsOk()) << "Couldn't initialize Sandboxed API"; + ZopfliApi api = ZopfliApi(&sandbox); + + std::string infile_s = GetTestFilePath("text"); + std::string outfile_s = GetTemporaryFile("text.out"); + ASSERT_THAT(outfile_s, Not(IsEmpty())); + + sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY)); + ASSERT_GE(infd.GetValue(), 0); + + sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY)); + ASSERT_GE(outfd.GetValue(), 0); + + absl::Status status = CompressFD(api, infd, outfd, GetParam()); + ASSERT_THAT(status, IsOk()) << "Unable to compress file"; + + off_t inpos = lseek(infd.GetValue(), 0, SEEK_END); + EXPECT_GE(inpos, 0); + + off_t outpos = lseek(outfd.GetValue(), 0, SEEK_END); + EXPECT_GE(outpos, 0); + + EXPECT_LT(outpos, inpos); +} + +INSTANTIATE_TEST_SUITE_P(SandboxTest, TestTextFD, + testing::Values(ZOPFLI_FORMAT_DEFLATE, + ZOPFLI_FORMAT_GZIP, + ZOPFLI_FORMAT_ZLIB)); + +TEST_P(TestBinaryFD, Compress) { + ZopfliSapiSandbox sandbox; + ASSERT_THAT(sandbox.Init(), IsOk()) << "Couldn't initialize Sandboxed API"; + ZopfliApi api = ZopfliApi(&sandbox); + + std::string infile_s = GetTestFilePath("binary"); + std::string outfile_s = GetTemporaryFile("binary.out"); + ASSERT_THAT(outfile_s, Not(IsEmpty())); + + sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY)); + ASSERT_GE(infd.GetValue(), 0); + + sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY)); + ASSERT_GE(outfd.GetValue(), 0); + + absl::Status status = CompressFD(api, infd, outfd, GetParam()); + ASSERT_THAT(status, IsOk()) << "Unable to compress file"; + + off_t inpos = lseek(infd.GetValue(), 0, SEEK_END); + EXPECT_GE(inpos, 0); + + off_t outpos = lseek(outfd.GetValue(), 0, SEEK_END); + EXPECT_GE(outpos, 0); + + EXPECT_LT(outpos, inpos); +} + +INSTANTIATE_TEST_SUITE_P(SandboxTest, TestBinaryFD, + testing::Values(ZOPFLI_FORMAT_DEFLATE, + ZOPFLI_FORMAT_GZIP, + ZOPFLI_FORMAT_ZLIB)); } // namespace diff --git a/contrib/zopfli/utils/utils_zopfli.cc b/contrib/zopfli/utils/utils_zopfli.cc index ef41b18..1a79b01 100644 --- a/contrib/zopfli/utils/utils_zopfli.cc +++ b/contrib/zopfli/utils/utils_zopfli.cc @@ -54,3 +54,25 @@ absl::Status Compress(ZopfliApi& api, std::ifstream& instream, } return absl::OkStatus(); } + +absl::Status CompressFD(ZopfliApi& api, sapi::v::Fd& infd, sapi::v::Fd& outfd, + ZopfliFormat format) { + SAPI_RETURN_IF_ERROR(api.GetSandbox()->TransferToSandboxee(&infd)); + SAPI_RETURN_IF_ERROR(api.GetSandbox()->TransferToSandboxee(&outfd)); + + sapi::v::Struct options; + SAPI_RETURN_IF_ERROR(api.ZopfliInitOptions(options.PtrAfter())); + + SAPI_ASSIGN_OR_RETURN( + int ret, api.ZopfliCompressFD(options.PtrBefore(), format, + infd.GetRemoteFd(), outfd.GetRemoteFd())); + + infd.CloseRemoteFd(api.GetSandbox()->rpc_channel()).IgnoreError(); + outfd.CloseRemoteFd(api.GetSandbox()->rpc_channel()).IgnoreError(); + + if (ret == -1) { + return absl::UnavailableError("Unable to compress file"); + } + + return absl::OkStatus(); +} diff --git a/contrib/zopfli/utils/utils_zopfli.h b/contrib/zopfli/utils/utils_zopfli.h index 664e861..5095808 100644 --- a/contrib/zopfli/utils/utils_zopfli.h +++ b/contrib/zopfli/utils/utils_zopfli.h @@ -23,4 +23,7 @@ absl::Status Compress(ZopfliApi& api, std::ifstream& instream, std::ofstream& outstream, ZopfliFormat format); +absl::Status CompressFD(ZopfliApi& api, sapi::v::Fd& infd, sapi::v::Fd& outfd, + ZopfliFormat format); + #endif // CONTRIB_ZOPFLI_UTILS_UTILS_ZOPFLI_H_ diff --git a/contrib/zopfli/wrapper/CMakeLists.txt b/contrib/zopfli/wrapper/CMakeLists.txt new file mode 100644 index 0000000..a9f23df --- /dev/null +++ b/contrib/zopfli/wrapper/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_library( + wrapper_zopfli STATIC + + wrapper_zopfli.cc +) + +target_link_libraries(wrapper_zopfli PUBLIC + Zopfli::libzopfli +) + +target_include_directories(wrapper_zopfli PUBLIC + ${SAPI_SOURCE_DIR} + ${libzopfli_SOURCE_DIR}/src/zopfli +) diff --git a/contrib/zopfli/wrapper/wrapper_zopfli.cc b/contrib/zopfli/wrapper/wrapper_zopfli.cc new file mode 100644 index 0000000..ccc9dea --- /dev/null +++ b/contrib/zopfli/wrapper/wrapper_zopfli.cc @@ -0,0 +1,50 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "contrib/zopfli/wrapper/wrapper_zopfli.h" + +#include +#include + +#include +#include +#include + +int ZopfliCompressFD(const ZopfliOptions* options, ZopfliFormat output_type, + int infd, int outfd) { + off_t insize = lseek(infd, 0, SEEK_END); + if (insize < 0) { + return -1; + } + if (lseek(infd, 0, SEEK_SET) < 0) { + return -1; + } + + auto inbuf = std::make_unique(insize); + if (read(infd, inbuf.get(), insize) != insize) { + return -1; + } + + size_t outsize = 0; + uint8_t* outbuf = nullptr; + ZopfliCompress(options, output_type, inbuf.get(), insize, &outbuf, &outsize); + size_t retsize = write(outfd, outbuf, outsize); + free(outbuf); + + if (outsize != retsize) { + return -1; + } + + return 0; +} diff --git a/contrib/zopfli/wrapper/wrapper_zopfli.h b/contrib/zopfli/wrapper/wrapper_zopfli.h new file mode 100644 index 0000000..0bb02fe --- /dev/null +++ b/contrib/zopfli/wrapper/wrapper_zopfli.h @@ -0,0 +1,25 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_ +#define CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_ + +#include "zopfli.h" + +extern "C" { +int ZopfliCompressFD(const ZopfliOptions* options, ZopfliFormat output_type, + int infd, int outfd); +}; + +#endif // CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_