work in progress

source/detail/cryptography/compound_document.cpp

@@ -91,6 +91,10 @@ std::vector<std::u16string> split_path(const std::u16string &path_string)
 }
 
 const sector_id FreeSector = -1;
+const sector_id EndOfChain = -2;
+const sector_id SATSector = -3;
+const sector_id MSATSector = -4;
+
 const directory_id End = -1;
 
 } // namespace
@@ -423,6 +427,8 @@ compound_document::compound_document(std::vector<std::uint8_t> &data)
       rb_tree_(new red_black_tree(entries_))
 {
     header_ = compound_document_header();
+    header_.msat.fill(FreeSector);
+    msat_.resize(109, FreeSector);
     header_.directory_start = allocate_sectors(1);
 
     write_header();
@@ -460,7 +466,6 @@ std::size_t compound_document::short_sector_size()
     return static_cast<std::size_t>(1) << header_.short_sector_size_power;
 }
 
-
 std::vector<std::uint8_t> compound_document::read_stream(const std::u16string &name)
 {
     const auto &entry = find_entry(name);
@@ -475,14 +480,24 @@ std::vector<std::uint8_t> compound_document::read_stream(const std::u16string &n
 
 void compound_document::write_stream(const std::u16string &name, const std::vector<std::uint8_t> &data)
 {
-    const auto num_sectors = data.size() / sector_size() + (data.size() % sector_size() ? 1 : 0);
-
     auto &entry = contains_entry(name)
         ? find_entry(name)
         : insert_entry(name, compound_document_entry::entry_type::UserStream);
-    entry.start = allocate_sectors(num_sectors);
+    entry.size = data.size();
+
+    if (entry.size < header_.threshold)
+    {
+        const auto num_sectors = data.size() / short_sector_size() + (data.size() % short_sector_size() ? 1 : 0);
+        entry.start = allocate_short_sectors(num_sectors);
+        write_short(data, entry.start);
+    }
+    else
+    {
+        const auto num_sectors = data.size() / sector_size() + (data.size() % sector_size() ? 1 : 0);
+        entry.start = allocate_sectors(num_sectors);
+        write(data, entry.start);
+    }
 
-    write(data, entry.start);
     write_directory_tree();
 }
 
@@ -502,17 +517,29 @@ void compound_document::write(const std::vector<byte> &data, sector_id start)
     }
 }
 
-void compound_document::write_short(const std::vector<byte> &data, sector_id start)
+void compound_document::write_short(const std::vector<byte> &data, sector_id current_short)
 {
-    const auto num_sectors = data.size() / sector_size() + (data.size() % sector_size() ? 1 : 0);
+    const auto header_size = sizeof(compound_document_header);
-    auto current_sector = start;
+    auto sector_data_reader = binary_reader(data);
+    auto first_short_sector = find_entry(u"Root Entry").start;
 
-    for (auto i = std::size_t(0); i < num_sectors; ++i)
+    while (current_short >= 0)
     {
-        auto position = sector_size() * current_sector;
+        auto short_position = static_cast<std::size_t>(short_sector_size() * current_short);
-        auto current_sector_size = data.size() % sector_size();
+        auto current_sector_index = 0;
-        writer_->append(data, position, current_sector_size);
+        auto current_sector = first_short_sector;
-        current_sector = ssat_[current_sector];
+
+        while (current_sector_index < short_position / sector_size())
+        {
+            current_sector = sat_[current_sector];
+            ++current_sector_index;
+        }
+
+        auto offset = short_position % sector_size();
+        writer_->offset(header_size + sector_size() * current_sector + offset);
+        writer_->append(data, offset, short_sector_size());
+
+        current_short = ssat_[current_short];
     }
 }
 
@@ -611,7 +638,7 @@ void compound_document::read_sat()
 void compound_document::read_ssat()
 {
     ssat_.clear();
-    auto current = header_.short_table_start;
+    auto current = header_.ssat_start;
 
     while (current >= 0)
     {
@@ -629,38 +656,29 @@ void compound_document::read_ssat()
 
 sector_id compound_document::allocate_sectors(std::size_t count)
 {
-    const auto sector_data_start = sizeof(compound_document_header);
     const auto sectors_per_sector = sector_size() / sizeof(sector_id);
     auto num_free = static_cast<std::size_t>(std::count(sat_.begin(), sat_.end(), FreeSector));
 
-    if (num_free < count)
+    while (num_free < count)
     {
-        // increase size of allocation table, plus extra sector for sat table
+        const auto sat_index = sat_.size() / sectors_per_sector;
-        auto new_size = sat_.size() + count - num_free + 1;
+        const auto msat_index = std::find(msat_.begin(), msat_.end(), FreeSector) - msat_.begin();
-        new_size = (new_size / sectors_per_sector 
+
-            + (new_size % sectors_per_sector != 0 ? 1 : 0)) * sectors_per_sector;
+        msat_[msat_index] = sat_index;
-        sat_.resize(new_size, FreeSector);
+        header_.msat[msat_index] = sat_index;
 
-        // allocate new sat sector
-        auto new_sat_sector = allocate_sectors(1);
-        msat_.push_back(new_sat_sector);
         ++header_.num_msat_sectors;
 
-        writer_->offset(sector_data_start + new_sat_sector * sector_size());
+        write_header();
-        writer_->append(sat_, sat_.size() - sectors_per_sector, sectors_per_sector);
 
-        if (msat_.size() > std::size_t(109))
+        auto previous_size = sat_.size();
-        {
+        sat_.resize(sat_.size() + sectors_per_sector, FreeSector);
-            // allocate extra msat sector
+        sat_[sat_index] = SATSector;
-            ++header_.num_extra_msat_sectors;
+
-            auto new_msat_sector = allocate_sectors(1);
+        writer_->offset(sector_data_start() + msat_index * sectors_per_sector);
-            writer_->offset(sector_data_start + new_msat_sector * sector_size());
+        writer_->append(sat_, previous_size, sectors_per_sector);
-            writer_->write(new_msat_sector);
+
-        }
+        num_free = static_cast<std::size_t>(std::count(sat_.begin(), sat_.end(), FreeSector));
-        else
-        {
-            header_.msat.at(msat_.size() - 1) = new_sat_sector;
-        }
     }
 
     auto allocated = 0;
@@ -672,9 +690,14 @@ sector_id compound_document::allocate_sectors(std::size_t count)
     {
         const auto next_iter = std::find(sat_.begin() + current + 1, sat_.end(), FreeSector);
         const auto next = sector_id(next_iter - sat_.begin());
-        sat_[current] = (allocated == count - 1) ? -2 : next;
 
-        if (sector_data_start + (current + 1) * sector_size() > writer_->size())
+        sat_[current] = (allocated == count - 1) ? EndOfChain : next;
+
+        auto msat_index = current / sectors_per_sector;
+        writer_->offset(sector_data_start() + msat_index * sector_size() + current * sizeof(sector_id));
+        writer_->write(sat_[current]);
+
+        if (sector_data_start() + (current + 1) * sector_size() > writer_->size())
         {
             writer_->extend(sector_size());
         }
@@ -692,7 +715,7 @@ void compound_document::reallocate_sectors(sector_id start, std::size_t count)
 
     for (auto i = std::size_t(0); i < count; ++i)
     {
-        if (current_sector == -2)
+        if (current_sector == EndOfChain)
         {
             sat_[current_sector] = allocate_sectors(count - i);
         }
@@ -701,6 +724,67 @@ void compound_document::reallocate_sectors(sector_id start, std::size_t count)
     }
 }
 
+sector_chain compound_document::follow_chain(sector_id start)
+{
+    auto chain = sector_chain();
+    chain.push_back(start);
+
+    while (start != EndOfChain)
+    {
+        start = sat_[start];
+    }
+
+    return chain;
+}
+
+sector_id compound_document::allocate_short_sectors(std::size_t count)
+{
+    const auto sectors_per_sector = sector_size() / sizeof(sector_id);
+    const auto num_free = std::count(ssat_.begin(), ssat_.end(), FreeSector);
+    auto next_sat = EndOfChain;
+
+    if (num_free < count)
+    {
+        const auto num_to_allocate = (count - num_free) / sectors_per_sector + (count - num_free ? 1 : 0);
+        next_sat = allocate_sectors(num_to_allocate);
+
+        if (header_.num_short_sectors == 0)
+        {
+            header_.ssat_start = next_sat;
+        }
+
+        ssat_.resize(ssat_.size() + sectors_per_sector * num_to_allocate, FreeSector);
+    }
+
+    header_.num_short_sectors += count;
+    write_header();
+
+    const auto start_iter = std::find(ssat_.begin(), ssat_.end(), FreeSector);
+    const auto start = sector_id(start_iter - ssat_.begin());
+    auto current = start;
+    auto allocated = std::size_t(0);
+    auto current_sat = header_.ssat_start;
+
+    while (allocated < count)
+    {
+        const auto next_iter = std::find(ssat_.begin() + current + 1, ssat_.end(), FreeSector);
+        const auto next = sector_id(next_iter - ssat_.begin());
+
+        ssat_[current] = (allocated == count - 1) ? EndOfChain : next;
+
+        current = next;
+        ++allocated;
+    }
+
+    if (find_entry(u"Root Entry").start < 0)
+    {
+        find_entry(u"Root Entry").start = start;
+    }
+
+    return start;
+
+}
+
 compound_document_entry &compound_document::insert_entry(
     const std::u16string &name,
     compound_document_entry::entry_type type)
@@ -752,10 +836,10 @@ void compound_document::write_directory_tree()
     auto entry_index = directory_id(0);
 
     reallocate_sectors(header_.directory_start, required_sectors);
+    writer_->offset(sector_data_start() + current_sector_id * sector_size());
 
     for (auto &e : entries_)
     {
-        writer_->offset(sector_data_start() + current_sector_id * sector_size());
         writer_->write(e);
 
         ++entry_index;
@@ -763,6 +847,7 @@ void compound_document::write_directory_tree()
         if (entry_index % entries_per_sector == 0)
         {
             current_sector_id = sat_[current_sector_id];
+            writer_->offset(sector_data_start() + current_sector_id * sector_size());
         }        
     }
 }

source/detail/cryptography/compound_document.hpp

@@ -47,7 +47,7 @@ struct compound_document_header
         little_endian = 0xFEFF
     };
 
-    std::uint64_t file_id = 0xe11ab1a1e011cfd0;
+    std::uint64_t file_id = 0xE11AB1A1E011CFD0;
     std::array<std::uint8_t, 16> ignore1 = { { 0 } };
     std::uint16_t revision = 0x003E;
     std::uint16_t version = 0x0003;
@@ -59,7 +59,7 @@ struct compound_document_header
     sector_id directory_start = -1;
     std::array<std::uint8_t, 4> ignore3 = { { 0 } };
     std::uint32_t threshold = 4096;
-    sector_id short_table_start = -1;
+    sector_id ssat_start = -2;
     std::uint32_t num_short_sectors = 0;
     sector_id extra_msat_start = -2;
     std::uint32_t num_extra_msat_sectors = 0;
@@ -99,14 +99,14 @@ struct compound_document_entry
     };
 
     std::array<char16_t, 32> name_array = { { 0 } };
-    std::uint16_t name_length = 0;
+    std::uint16_t name_length = 2;
     entry_type type = entry_type::Empty;
-    entry_color color = entry_color::Red;
+    entry_color color = entry_color::Black;
     directory_id prev = -1;
     directory_id next = -1;
     directory_id child = -1;
     std::array<std::uint8_t, 36> ignore;
-    sector_id start = -1;
+    sector_id start = -2;
     std::uint32_t size = 0;
     std::uint32_t ignore2;
 };
@@ -134,6 +134,8 @@ private:
     std::vector<byte> read(sector_id start);
     std::vector<byte> read_short(sector_id start);
 
+    sector_chain follow_chain(sector_id start);
+
     void read_msat();
     void read_sat();
     void read_ssat();
@@ -143,9 +145,6 @@ private:
     void write(const std::vector<byte> &data, sector_id start);
     void write_short(const std::vector<byte> &data, sector_id start);
 
-    void write_msat();
-    void write_sat();
-    void write_ssat();
     void write_header();
     void write_directory_tree();
 
@@ -153,7 +152,7 @@ private:
 
     sector_id allocate_sectors(std::size_t sectors);
     void reallocate_sectors(sector_id start, std::size_t sectors);
-    std::size_t allocated_sectors(sector_id start);
+    sector_id allocate_short_sectors(std::size_t sectors);
 
     compound_document_entry &insert_entry(const std::u16string &path, 
         compound_document_entry::entry_type type);

source/detail/serialization/vector_streambuf.hpp

@@ -262,10 +262,11 @@ private:
 /// </summary>
 XLNT_API inline std::vector<std::uint8_t> to_vector(std::istream &in_stream)
 {
-    std::vector<std::uint8_t> bytes;
+    in_stream.seekg(0, std::ios::end);
-    vector_ostreambuf buffer(bytes);
+    std::vector<std::uint8_t> bytes(in_stream.tellg(), 0);
-    std::ostream out_stream(&buffer);
+    in_stream.seekg(0, std::ios::beg);
-    out_stream << in_stream.rdbuf();
+    in_stream.read(reinterpret_cast<char *>(bytes.data()), bytes.size());
+
     return bytes;
 }
 
@@ -274,8 +275,7 @@ XLNT_API inline std::vector<std::uint8_t> to_vector(std::istream &in_stream)
 /// </summary>
 XLNT_API inline void to_stream(const std::vector<std::uint8_t> &bytes, std::ostream &out_stream)
 {
-    vector_istreambuf buffer(bytes);
+    out_stream.write(reinterpret_cast<const char *>(bytes.data()), bytes.size());
-    out_stream << &buffer;
 }
 
 /// <summary>

tests/runner.cpp

@@ -45,6 +45,8 @@
 #include <worksheet/range_test_suite.hpp>
 #include <worksheet/worksheet_test_suite.hpp>
 
+#include <detail/cryptography/compound_document.hpp>
+
 test_status overall_status;
 
 template<typename T>
@@ -71,6 +73,13 @@ void print_summary()
 
 int main()
 {
+    std::vector<std::uint8_t> bytes;
+    xlnt::detail::compound_document doc(bytes);
+    doc.write_stream(u"aaa", std::vector<std::uint8_t>(4095, 'a'));
+    doc.write_stream(u"bbb", std::vector<std::uint8_t>(4095, 'b'));
+    doc.write_stream(u"ccc", std::vector<std::uint8_t>(4095, 'c'));
+    xlnt::detail::to_stream(bytes, std::ofstream("cd.xlsx", std::ios::binary));
+
     // cell
     run_tests<cell_test_suite>();
     run_tests<index_types_test_suite>();