From 05102f2c41b1e936a513fe3d8dc60ab02ddec6bf Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Wed, 2 Feb 2022 12:34:05 -0500 Subject: [PATCH 1/5] ZStandard: use seprate structure for (de)compression --- contrib/zstd/CMakeLists.txt | 3 +++ contrib/zstd/utils/utils_zstd.cc | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/contrib/zstd/CMakeLists.txt b/contrib/zstd/CMakeLists.txt index c045933..4a35f22 100644 --- a/contrib/zstd/CMakeLists.txt +++ b/contrib/zstd/CMakeLists.txt @@ -51,6 +51,9 @@ add_sapi_library( ZSTD_createDCtx ZSTD_freeDCtx + ZSTD_createCCtx + ZSTD_freeCCtx + ZSTD_CCtx_setParameter ZSTD_compressBound diff --git a/contrib/zstd/utils/utils_zstd.cc b/contrib/zstd/utils/utils_zstd.cc index ab148f6..e838d44 100644 --- a/contrib/zstd/utils/utils_zstd.cc +++ b/contrib/zstd/utils/utils_zstd.cc @@ -110,16 +110,16 @@ absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, } // Create Zstd context. - SAPI_ASSIGN_OR_RETURN(ZSTD_DCtx * dctx, api.ZSTD_createDCtx()); - sapi::v::RemotePtr rdctx(dctx); + SAPI_ASSIGN_OR_RETURN(ZSTD_CCtx * cctx, api.ZSTD_createCCtx()); + sapi::v::RemotePtr rcctx(cctx); SAPI_ASSIGN_OR_RETURN(iserr, api.ZSTD_CCtx_setParameter( - &rdctx, ZSTD_c_compressionLevel, level)); + &rcctx, ZSTD_c_compressionLevel, level)); if (!iserr) { return absl::UnavailableError("Unable to set parameter"); } SAPI_ASSIGN_OR_RETURN( - iserr, api.ZSTD_CCtx_setParameter(&rdctx, ZSTD_c_checksumFlag, 1)); + iserr, api.ZSTD_CCtx_setParameter(&rcctx, ZSTD_c_checksumFlag, 1)); if (!iserr) { return absl::UnavailableError("Unable to set parameter"); } @@ -151,7 +151,7 @@ absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, struct_out.mutable_data()->size = outbuf.GetSize(); SAPI_ASSIGN_OR_RETURN(size_t remaining, api.ZSTD_compressStream2( - &rdctx, struct_out.PtrBoth(), + &rcctx, struct_out.PtrBoth(), struct_in.PtrBoth(), mode)); SAPI_ASSIGN_OR_RETURN(int iserr, api.ZSTD_isError(remaining)) if (iserr) { @@ -175,7 +175,7 @@ absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, } } - api.ZSTD_freeDCtx(&rdctx).IgnoreError(); + api.ZSTD_freeDCtx(&rcctx).IgnoreError(); return absl::OkStatus(); } From 7cb3f4a04dc2c68c46e5eb0ee6260101d3f9518b Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Wed, 2 Feb 2022 12:34:23 -0500 Subject: [PATCH 2/5] ZStandard: deduplicate the list of functions The ZSTD_compressBound is twice on the list. --- contrib/zstd/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/zstd/CMakeLists.txt b/contrib/zstd/CMakeLists.txt index 4a35f22..bae244f 100644 --- a/contrib/zstd/CMakeLists.txt +++ b/contrib/zstd/CMakeLists.txt @@ -56,7 +56,6 @@ add_sapi_library( ZSTD_CCtx_setParameter - ZSTD_compressBound ZSTD_compress ZSTD_compressStream ZSTD_compressStream2 From d0ce037fa20f2040b2d82e849f650f55b961ddfa Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Wed, 2 Feb 2022 13:34:14 -0500 Subject: [PATCH 3/5] ZStandard: Add additional check of error code --- contrib/zstd/utils/utils_zstd.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/zstd/utils/utils_zstd.cc b/contrib/zstd/utils/utils_zstd.cc index e838d44..9a8936f 100644 --- a/contrib/zstd/utils/utils_zstd.cc +++ b/contrib/zstd/utils/utils_zstd.cc @@ -115,12 +115,14 @@ absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, SAPI_ASSIGN_OR_RETURN(iserr, api.ZSTD_CCtx_setParameter( &rcctx, ZSTD_c_compressionLevel, level)); - if (!iserr) { + SAPI_ASSIGN_OR_RETURN(iserr, api.ZSTD_isError(iserr)) + if (iserr) { return absl::UnavailableError("Unable to set parameter"); } SAPI_ASSIGN_OR_RETURN( iserr, api.ZSTD_CCtx_setParameter(&rcctx, ZSTD_c_checksumFlag, 1)); - if (!iserr) { + SAPI_ASSIGN_OR_RETURN(iserr, api.ZSTD_isError(iserr)) + if (iserr) { return absl::UnavailableError("Unable to set parameter"); } From 74c32625bf261efa91b51b379f43926104a3d17c Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Wed, 2 Feb 2022 15:45:58 -0500 Subject: [PATCH 4/5] ZStandard: fix descirption of test --- contrib/zstd/test/zstd_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/zstd/test/zstd_test.cc b/contrib/zstd/test/zstd_test.cc index 17ce6f3..b997574 100644 --- a/contrib/zstd/test/zstd_test.cc +++ b/contrib/zstd/test/zstd_test.cc @@ -207,7 +207,7 @@ TEST(SandboxTest, CheckCompressStream) { ASSERT_TRUE(outfile.is_open()); absl::Status status = CompressStream(api, infile, outfile, 0); - ASSERT_THAT(status, IsOk()) << "Unable to compress file in memory"; + ASSERT_THAT(status, IsOk()) << "Unable to compress stream"; infile.clear(); @@ -233,7 +233,7 @@ TEST(SandboxTest, CheckDecompressStream) { ASSERT_TRUE(outfile.is_open()); absl::Status status = DecompressStream(api, infile, outfile); - ASSERT_THAT(status, IsOk()) << "Unable to decompress file in memory"; + ASSERT_THAT(status, IsOk()) << "Unable to decompress stream"; ASSERT_GT(outfile.tellp(), infile.tellg()); @@ -266,7 +266,7 @@ TEST(SandboxTest, CheckCompressAndDecompressStream) { ASSERT_TRUE(outmiddle.is_open()); status = CompressStream(api, infile, outmiddle, 0); - ASSERT_THAT(status, IsOk()) << "Unable to compress file in memory"; + ASSERT_THAT(status, IsOk()) << "Unable to compress stream"; infile.clear(); ASSERT_LT(outmiddle.tellp(), infile.tellg()); From d2a8c8a1eafb411a58ca8c7094b76adeb1170c1e Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Wed, 2 Feb 2022 16:00:33 -0500 Subject: [PATCH 5/5] ZStandard: minor style fixes --- contrib/zstd/utils/utils_zstd.cc | 38 ++++++++++++++++---------------- contrib/zstd/utils/utils_zstd.h | 16 +++++++------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/contrib/zstd/utils/utils_zstd.cc b/contrib/zstd/utils/utils_zstd.cc index 9a8936f..806b3d0 100644 --- a/contrib/zstd/utils/utils_zstd.cc +++ b/contrib/zstd/utils/utils_zstd.cc @@ -28,12 +28,12 @@ std::streamsize GetStreamSize(std::ifstream& stream) { return ssize; } -absl::Status CompressInMemory(ZstdApi& api, std::ifstream& in_file, +absl::Status CompressInMemory(ZstdApi& api, std::ifstream& in_stream, std::ofstream& out_stream, int level) { - std::streamsize ssize = GetStreamSize(in_file); + std::streamsize ssize = GetStreamSize(in_stream); sapi::v::Array inbuf(ssize); - in_file.read(reinterpret_cast(inbuf.GetData()), ssize); - if (in_file.gcount() != ssize) { + in_stream.read(reinterpret_cast(inbuf.GetData()), ssize); + if (in_stream.gcount() != ssize) { return absl::UnavailableError("Unable to read file"); } @@ -57,13 +57,13 @@ absl::Status CompressInMemory(ZstdApi& api, std::ifstream& in_file, return absl::OkStatus(); } -absl::Status DecompressInMemory(ZstdApi& api, std::ifstream& in_file, +absl::Status DecompressInMemory(ZstdApi& api, std::ifstream& in_stream, std::ofstream& out_stream) { int iserr; - std::streamsize ssize = GetStreamSize(in_file); + std::streamsize ssize = GetStreamSize(in_stream); sapi::v::Array inbuf(ssize); - in_file.read(reinterpret_cast(inbuf.GetData()), ssize); - if (in_file.gcount() != ssize) { + in_stream.read(reinterpret_cast(inbuf.GetData()), ssize); + if (in_stream.gcount() != ssize) { return absl::UnavailableError("Unable to read file"); } @@ -94,7 +94,7 @@ absl::Status DecompressInMemory(ZstdApi& api, std::ifstream& in_file, return absl::OkStatus(); } -absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, +absl::Status CompressStream(ZstdApi& api, std::ifstream& in_stream, std::ofstream& out_stream, int level) { int iserr; @@ -127,8 +127,8 @@ absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, } // Compress. - while (in_file) { - in_file.read(reinterpret_cast(inbuf.GetData()), inbuf_size); + while (in_stream) { + in_stream.read(reinterpret_cast(inbuf.GetData()), inbuf_size); if (!api.GetSandbox()->TransferToSandboxee(&inbuf).ok()) { return absl::UnavailableError("Unable to transfer data"); @@ -137,10 +137,10 @@ absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, sapi::v::Struct struct_in; struct_in.mutable_data()->src = static_cast(inbuf.GetRemote()); struct_in.mutable_data()->pos = 0; - struct_in.mutable_data()->size = in_file.gcount(); + struct_in.mutable_data()->size = in_stream.gcount(); ZSTD_EndDirective mode = ZSTD_e_continue; - if (in_file.gcount() < inbuf_size) { + if (in_stream.gcount() < inbuf_size) { mode = ZSTD_e_end; } @@ -170,7 +170,7 @@ absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, } if (mode == ZSTD_e_continue) { - isdone = (struct_in.mutable_data()->pos == in_file.gcount()); + isdone = (struct_in.mutable_data()->pos == in_stream.gcount()); } else { isdone = (remaining == 0); } @@ -182,7 +182,7 @@ absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, return absl::OkStatus(); } -absl::Status DecompressStream(ZstdApi& api, std::ifstream& in_file, +absl::Status DecompressStream(ZstdApi& api, std::ifstream& in_stream, std::ofstream& out_stream) { // Create necessary buffers. SAPI_ASSIGN_OR_RETURN(size_t inbuf_size, api.ZSTD_CStreamInSize()); @@ -200,8 +200,8 @@ absl::Status DecompressStream(ZstdApi& api, std::ifstream& in_file, sapi::v::RemotePtr rdctx(dctx); // Decompress. - while (in_file) { - in_file.read(reinterpret_cast(inbuf.GetData()), inbuf_size); + while (in_stream) { + in_stream.read(reinterpret_cast(inbuf.GetData()), inbuf_size); if (!api.GetSandbox()->TransferToSandboxee(&inbuf).ok()) { return absl::UnavailableError("Unable to transfer data"); @@ -209,10 +209,10 @@ absl::Status DecompressStream(ZstdApi& api, std::ifstream& in_file, sapi::v::Struct struct_in; *struct_in.mutable_data() = {static_cast(inbuf.GetRemote()), - (size_t)in_file.gcount(), 0}; + (size_t)in_stream.gcount(), 0}; bool isdone = false; - while (struct_in.mutable_data()->pos < in_file.gcount()) { + while (struct_in.mutable_data()->pos < in_stream.gcount()) { sapi::v::Struct struct_out; *struct_out.mutable_data() = {static_cast(outbuf.GetRemote()), (size_t)outbuf.GetSize(), 0}; diff --git a/contrib/zstd/utils/utils_zstd.h b/contrib/zstd/utils/utils_zstd.h index 1c5f131..cac7a9d 100644 --- a/contrib/zstd/utils/utils_zstd.h +++ b/contrib/zstd/utils/utils_zstd.h @@ -20,14 +20,14 @@ #include "contrib/zstd/sandboxed.h" -absl::Status CompressInMemory(ZstdApi& api, std::ifstream& in_file, - std::ofstream& out_file, int level); -absl::Status DecompressInMemory(ZstdApi& api, std::ifstream& in_file, - std::ofstream& out_file); +absl::Status CompressInMemory(ZstdApi& api, std::ifstream& in_stream, + std::ofstream& out_stream, int level); +absl::Status DecompressInMemory(ZstdApi& api, std::ifstream& in_stream, + std::ofstream& out_stream); -absl::Status CompressStream(ZstdApi& api, std::ifstream& in_file, - std::ofstream& out_file, int level); -absl::Status DecompressStream(ZstdApi& api, std::ifstream& in_file, - std::ofstream& out_file); +absl::Status CompressStream(ZstdApi& api, std::ifstream& in_stream, + std::ofstream& out_stream, int level); +absl::Status DecompressStream(ZstdApi& api, std::ifstream& in_stream, + std::ofstream& out_stream); #endif // CONTRIB_ZSTD_UTILS_UTILS_ZSTD_H_