mirror of
https://github.com/google/sandboxed-api.git
synced 2024-03-22 13:11:30 +08:00
zopfli: introduce a wrapper
The goal is to use a file descriptor as an input. Thanks to that we shouldn't send a chunk of memory over expensive protocol.
This commit is contained in:
parent
dc03c38df1
commit
b3aca9ef48
|
@ -32,6 +32,8 @@ FetchContent_Declare(zopfli
|
||||||
)
|
)
|
||||||
FetchContent_MakeAvailable(zopfli)
|
FetchContent_MakeAvailable(zopfli)
|
||||||
|
|
||||||
|
add_subdirectory(wrapper)
|
||||||
|
|
||||||
add_sapi_library(
|
add_sapi_library(
|
||||||
sapi_zopfli
|
sapi_zopfli
|
||||||
|
|
||||||
|
@ -44,12 +46,14 @@ add_sapi_library(
|
||||||
ZopfliZlibCompress
|
ZopfliZlibCompress
|
||||||
ZopfliGzipCompress
|
ZopfliGzipCompress
|
||||||
|
|
||||||
|
ZopfliCompressFD
|
||||||
INPUTS
|
INPUTS
|
||||||
${zopfli_SOURCE_DIR}/src/zopfli/deflate.h
|
${zopfli_SOURCE_DIR}/src/zopfli/deflate.h
|
||||||
${zopfli_SOURCE_DIR}/src/zopfli/gzip_container.h
|
${zopfli_SOURCE_DIR}/src/zopfli/gzip_container.h
|
||||||
${zopfli_SOURCE_DIR}/src/zopfli/zlib_container.h
|
${zopfli_SOURCE_DIR}/src/zopfli/zlib_container.h
|
||||||
|
wrapper/wrapper_zopfli.h
|
||||||
|
|
||||||
LIBRARY Zopfli::libzopfli
|
LIBRARY wrapper_zopfli
|
||||||
LIBRARY_NAME Zopfli
|
LIBRARY_NAME Zopfli
|
||||||
NAMESPACE ""
|
NAMESPACE ""
|
||||||
)
|
)
|
||||||
|
|
|
@ -12,6 +12,7 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include <fcntl.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
|
@ -23,9 +24,39 @@
|
||||||
#include "contrib/zopfli/sandboxed.h"
|
#include "contrib/zopfli/sandboxed.h"
|
||||||
#include "contrib/zopfli/utils/utils_zopfli.h"
|
#include "contrib/zopfli/utils/utils_zopfli.h"
|
||||||
|
|
||||||
|
ABSL_FLAG(bool, stream, false, "stream memory to sandbox");
|
||||||
ABSL_FLAG(bool, zlib, false, "zlib compression");
|
ABSL_FLAG(bool, zlib, false, "zlib compression");
|
||||||
ABSL_FLAG(bool, gzip, false, "gzip compression");
|
ABSL_FLAG(bool, gzip, false, "gzip compression");
|
||||||
|
|
||||||
|
absl::Status CompressMain(ZopfliApi& api, std::string& infile_s,
|
||||||
|
std::string& outfile_s, ZopfliFormat format) {
|
||||||
|
std::ifstream infile(infile_s, std::ios::binary);
|
||||||
|
if (!infile.is_open()) {
|
||||||
|
return absl::UnavailableError(absl::StrCat("Unable to open ", infile_s));
|
||||||
|
}
|
||||||
|
std::ofstream outfile(outfile_s, std::ios::binary);
|
||||||
|
if (!outfile.is_open()) {
|
||||||
|
return absl::UnavailableError(absl::StrCat("Unable to open ", outfile_s));
|
||||||
|
}
|
||||||
|
|
||||||
|
return Compress(api, infile, outfile, format);
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status CompressMainFD(ZopfliApi& api, std::string& infile_s,
|
||||||
|
std::string& outfile_s, ZopfliFormat format) {
|
||||||
|
sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY));
|
||||||
|
if (infd.GetValue() < 0) {
|
||||||
|
return absl::UnavailableError(absl::StrCat("Unable to open ", infile_s));
|
||||||
|
}
|
||||||
|
|
||||||
|
sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY | O_CREAT));
|
||||||
|
if (outfd.GetValue() < 0) {
|
||||||
|
return absl::UnavailableError(absl::StrCat("Unable to open ", outfile_s));
|
||||||
|
}
|
||||||
|
|
||||||
|
return (CompressFD(api, infd, outfd, format));
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
int main(int argc, char* argv[]) {
|
||||||
std::string prog_name(argv[0]);
|
std::string prog_name(argv[0]);
|
||||||
google::InitGoogleLogging(argv[0]);
|
google::InitGoogleLogging(argv[0]);
|
||||||
|
@ -36,22 +67,13 @@ int main(int argc, char* argv[]) {
|
||||||
return EXIT_FAILURE;
|
return EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::ifstream infile(args[1], std::ios::binary);
|
|
||||||
if (!infile.is_open()) {
|
|
||||||
std::cerr << "Unable to open " << args[1] << std::endl;
|
|
||||||
return EXIT_FAILURE;
|
|
||||||
}
|
|
||||||
std::ofstream outfile(args[2], std::ios::binary);
|
|
||||||
if (!outfile.is_open()) {
|
|
||||||
std::cerr << "Unable to open " << args[2] << std::endl;
|
|
||||||
return EXIT_FAILURE;
|
|
||||||
}
|
|
||||||
|
|
||||||
ZopfliSapiSandbox sandbox;
|
ZopfliSapiSandbox sandbox;
|
||||||
if (!sandbox.Init().ok()) {
|
if (!sandbox.Init().ok()) {
|
||||||
std::cerr << "Unable to start sandbox\n";
|
std::cerr << "Unable to start sandbox\n";
|
||||||
return EXIT_FAILURE;
|
return EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
|
std::string infile_s(args[1]);
|
||||||
|
std::string outfile_s(args[2]);
|
||||||
|
|
||||||
ZopfliApi api(&sandbox);
|
ZopfliApi api(&sandbox);
|
||||||
|
|
||||||
|
@ -62,7 +84,13 @@ int main(int argc, char* argv[]) {
|
||||||
format = ZOPFLI_FORMAT_GZIP;
|
format = ZOPFLI_FORMAT_GZIP;
|
||||||
}
|
}
|
||||||
|
|
||||||
absl::Status status = Compress(api, infile, outfile, format);
|
absl::Status status;
|
||||||
|
if (absl::GetFlag(FLAGS_stream)) {
|
||||||
|
status = CompressMain(api, infile_s, outfile_s, format);
|
||||||
|
} else {
|
||||||
|
status = CompressMainFD(api, infile_s, outfile_s, format);
|
||||||
|
}
|
||||||
|
|
||||||
if (!status.ok()) {
|
if (!status.ok()) {
|
||||||
std::cerr << "Unable to compress file.\n";
|
std::cerr << "Unable to compress file.\n";
|
||||||
std::cerr << status << std::endl;
|
std::cerr << status << std::endl;
|
||||||
|
|
|
@ -28,12 +28,13 @@ class ZopfliSapiSandbox : public ZopfliSandbox {
|
||||||
std::unique_ptr<sandbox2::Policy> ModifyPolicy(
|
std::unique_ptr<sandbox2::Policy> ModifyPolicy(
|
||||||
sandbox2::PolicyBuilder *) override {
|
sandbox2::PolicyBuilder *) override {
|
||||||
return sandbox2::PolicyBuilder()
|
return sandbox2::PolicyBuilder()
|
||||||
.AllowStaticStartup()
|
.AllowDynamicStartup()
|
||||||
.AllowWrite()
|
.AllowWrite()
|
||||||
.AllowExit()
|
.AllowExit()
|
||||||
.AllowMmap()
|
.AllowMmap()
|
||||||
.AllowSystemMalloc()
|
.AllowSystemMalloc()
|
||||||
.AllowSyscalls({
|
.AllowSyscalls({
|
||||||
|
__NR_recvmsg,
|
||||||
__NR_sysinfo,
|
__NR_sysinfo,
|
||||||
})
|
})
|
||||||
#ifdef __NR_open
|
#ifdef __NR_open
|
||||||
|
|
|
@ -12,6 +12,8 @@
|
||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include <fcntl.h>
|
||||||
|
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
|
|
||||||
#include "contrib/zopfli/sandboxed.h"
|
#include "contrib/zopfli/sandboxed.h"
|
||||||
|
@ -41,8 +43,9 @@ std::string GetTemporaryFile(const std::string& filename) {
|
||||||
}
|
}
|
||||||
|
|
||||||
class TestText : public testing::TestWithParam<ZopfliFormat> {};
|
class TestText : public testing::TestWithParam<ZopfliFormat> {};
|
||||||
|
|
||||||
class TestBinary : public testing::TestWithParam<ZopfliFormat> {};
|
class TestBinary : public testing::TestWithParam<ZopfliFormat> {};
|
||||||
|
class TestTextFD : public testing::TestWithParam<ZopfliFormat> {};
|
||||||
|
class TestBinaryFD : public testing::TestWithParam<ZopfliFormat> {};
|
||||||
|
|
||||||
TEST_P(TestText, Compress) {
|
TEST_P(TestText, Compress) {
|
||||||
ZopfliSapiSandbox sandbox;
|
ZopfliSapiSandbox sandbox;
|
||||||
|
@ -95,4 +98,68 @@ INSTANTIATE_TEST_SUITE_P(SandboxTest, TestBinary,
|
||||||
testing::Values(ZOPFLI_FORMAT_DEFLATE,
|
testing::Values(ZOPFLI_FORMAT_DEFLATE,
|
||||||
ZOPFLI_FORMAT_GZIP,
|
ZOPFLI_FORMAT_GZIP,
|
||||||
ZOPFLI_FORMAT_ZLIB));
|
ZOPFLI_FORMAT_ZLIB));
|
||||||
|
|
||||||
|
TEST_P(TestTextFD, Compress) {
|
||||||
|
ZopfliSapiSandbox sandbox;
|
||||||
|
ASSERT_THAT(sandbox.Init(), IsOk()) << "Couldn't initialize Sandboxed API";
|
||||||
|
ZopfliApi api = ZopfliApi(&sandbox);
|
||||||
|
|
||||||
|
std::string infile_s = GetTestFilePath("text");
|
||||||
|
std::string outfile_s = GetTemporaryFile("text.out");
|
||||||
|
ASSERT_THAT(outfile_s, Not(IsEmpty()));
|
||||||
|
|
||||||
|
sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY));
|
||||||
|
ASSERT_GE(infd.GetValue(), 0);
|
||||||
|
|
||||||
|
sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY));
|
||||||
|
ASSERT_GE(outfd.GetValue(), 0);
|
||||||
|
|
||||||
|
absl::Status status = CompressFD(api, infd, outfd, GetParam());
|
||||||
|
ASSERT_THAT(status, IsOk()) << "Unable to compress file";
|
||||||
|
|
||||||
|
off_t inpos = lseek(infd.GetValue(), 0, SEEK_END);
|
||||||
|
EXPECT_GE(inpos, 0);
|
||||||
|
|
||||||
|
off_t outpos = lseek(outfd.GetValue(), 0, SEEK_END);
|
||||||
|
EXPECT_GE(outpos, 0);
|
||||||
|
|
||||||
|
EXPECT_LT(outpos, inpos);
|
||||||
|
}
|
||||||
|
|
||||||
|
INSTANTIATE_TEST_SUITE_P(SandboxTest, TestTextFD,
|
||||||
|
testing::Values(ZOPFLI_FORMAT_DEFLATE,
|
||||||
|
ZOPFLI_FORMAT_GZIP,
|
||||||
|
ZOPFLI_FORMAT_ZLIB));
|
||||||
|
|
||||||
|
TEST_P(TestBinaryFD, Compress) {
|
||||||
|
ZopfliSapiSandbox sandbox;
|
||||||
|
ASSERT_THAT(sandbox.Init(), IsOk()) << "Couldn't initialize Sandboxed API";
|
||||||
|
ZopfliApi api = ZopfliApi(&sandbox);
|
||||||
|
|
||||||
|
std::string infile_s = GetTestFilePath("binary");
|
||||||
|
std::string outfile_s = GetTemporaryFile("binary.out");
|
||||||
|
ASSERT_THAT(outfile_s, Not(IsEmpty()));
|
||||||
|
|
||||||
|
sapi::v::Fd infd(open(infile_s.c_str(), O_RDONLY));
|
||||||
|
ASSERT_GE(infd.GetValue(), 0);
|
||||||
|
|
||||||
|
sapi::v::Fd outfd(open(outfile_s.c_str(), O_WRONLY));
|
||||||
|
ASSERT_GE(outfd.GetValue(), 0);
|
||||||
|
|
||||||
|
absl::Status status = CompressFD(api, infd, outfd, GetParam());
|
||||||
|
ASSERT_THAT(status, IsOk()) << "Unable to compress file";
|
||||||
|
|
||||||
|
off_t inpos = lseek(infd.GetValue(), 0, SEEK_END);
|
||||||
|
EXPECT_GE(inpos, 0);
|
||||||
|
|
||||||
|
off_t outpos = lseek(outfd.GetValue(), 0, SEEK_END);
|
||||||
|
EXPECT_GE(outpos, 0);
|
||||||
|
|
||||||
|
EXPECT_LT(outpos, inpos);
|
||||||
|
}
|
||||||
|
|
||||||
|
INSTANTIATE_TEST_SUITE_P(SandboxTest, TestBinaryFD,
|
||||||
|
testing::Values(ZOPFLI_FORMAT_DEFLATE,
|
||||||
|
ZOPFLI_FORMAT_GZIP,
|
||||||
|
ZOPFLI_FORMAT_ZLIB));
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
|
@ -54,3 +54,25 @@ absl::Status Compress(ZopfliApi& api, std::ifstream& instream,
|
||||||
}
|
}
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
absl::Status CompressFD(ZopfliApi& api, sapi::v::Fd& infd, sapi::v::Fd& outfd,
|
||||||
|
ZopfliFormat format) {
|
||||||
|
SAPI_RETURN_IF_ERROR(api.GetSandbox()->TransferToSandboxee(&infd));
|
||||||
|
SAPI_RETURN_IF_ERROR(api.GetSandbox()->TransferToSandboxee(&outfd));
|
||||||
|
|
||||||
|
sapi::v::Struct<ZopfliOptions> options;
|
||||||
|
SAPI_RETURN_IF_ERROR(api.ZopfliInitOptions(options.PtrAfter()));
|
||||||
|
|
||||||
|
SAPI_ASSIGN_OR_RETURN(
|
||||||
|
int ret, api.ZopfliCompressFD(options.PtrBefore(), format,
|
||||||
|
infd.GetRemoteFd(), outfd.GetRemoteFd()));
|
||||||
|
|
||||||
|
infd.CloseRemoteFd(api.GetSandbox()->rpc_channel()).IgnoreError();
|
||||||
|
outfd.CloseRemoteFd(api.GetSandbox()->rpc_channel()).IgnoreError();
|
||||||
|
|
||||||
|
if (ret == -1) {
|
||||||
|
return absl::UnavailableError("Unable to compress file");
|
||||||
|
}
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
|
@ -23,4 +23,7 @@
|
||||||
absl::Status Compress(ZopfliApi& api, std::ifstream& instream,
|
absl::Status Compress(ZopfliApi& api, std::ifstream& instream,
|
||||||
std::ofstream& outstream, ZopfliFormat format);
|
std::ofstream& outstream, ZopfliFormat format);
|
||||||
|
|
||||||
|
absl::Status CompressFD(ZopfliApi& api, sapi::v::Fd& infd, sapi::v::Fd& outfd,
|
||||||
|
ZopfliFormat format);
|
||||||
|
|
||||||
#endif // CONTRIB_ZOPFLI_UTILS_UTILS_ZOPFLI_H_
|
#endif // CONTRIB_ZOPFLI_UTILS_UTILS_ZOPFLI_H_
|
||||||
|
|
28
contrib/zopfli/wrapper/CMakeLists.txt
Normal file
28
contrib/zopfli/wrapper/CMakeLists.txt
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
# Copyright 2022 Google LLC
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
add_library(
|
||||||
|
wrapper_zopfli STATIC
|
||||||
|
|
||||||
|
wrapper_zopfli.cc
|
||||||
|
)
|
||||||
|
|
||||||
|
target_link_libraries(wrapper_zopfli PUBLIC
|
||||||
|
Zopfli::libzopfli
|
||||||
|
)
|
||||||
|
|
||||||
|
target_include_directories(wrapper_zopfli PUBLIC
|
||||||
|
${SAPI_SOURCE_DIR}
|
||||||
|
${libzopfli_SOURCE_DIR}/src/zopfli
|
||||||
|
)
|
50
contrib/zopfli/wrapper/wrapper_zopfli.cc
Normal file
50
contrib/zopfli/wrapper/wrapper_zopfli.cc
Normal file
|
@ -0,0 +1,50 @@
|
||||||
|
// Copyright 2022 Google LLC
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "contrib/zopfli/wrapper/wrapper_zopfli.h"
|
||||||
|
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
#include <cstdio>
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
int ZopfliCompressFD(const ZopfliOptions* options, ZopfliFormat output_type,
|
||||||
|
int infd, int outfd) {
|
||||||
|
off_t insize = lseek(infd, 0, SEEK_END);
|
||||||
|
if (insize < 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (lseek(infd, 0, SEEK_SET) < 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto inbuf = std::make_unique<uint8_t[]>(insize);
|
||||||
|
if (read(infd, inbuf.get(), insize) != insize) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t outsize = 0;
|
||||||
|
uint8_t* outbuf = nullptr;
|
||||||
|
ZopfliCompress(options, output_type, inbuf.get(), insize, &outbuf, &outsize);
|
||||||
|
size_t retsize = write(outfd, outbuf, outsize);
|
||||||
|
free(outbuf);
|
||||||
|
|
||||||
|
if (outsize != retsize) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
25
contrib/zopfli/wrapper/wrapper_zopfli.h
Normal file
25
contrib/zopfli/wrapper/wrapper_zopfli.h
Normal file
|
@ -0,0 +1,25 @@
|
||||||
|
// Copyright 2022 Google LLC
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// https://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#ifndef CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_
|
||||||
|
#define CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_
|
||||||
|
|
||||||
|
#include "zopfli.h"
|
||||||
|
|
||||||
|
extern "C" {
|
||||||
|
int ZopfliCompressFD(const ZopfliOptions* options, ZopfliFormat output_type,
|
||||||
|
int infd, int outfd);
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif // CONTRIB_ZOPFLI_WRAPPER_WRAPPER_ZOPFLI_H_
|
Loading…
Reference in New Issue
Block a user