Updated sandbox construction logic and CMakeLists

More flexible CMake file with variables
Added logic to check whether proj.db exists and fetch it from the environment variable

oss-internship-2020/gdal/raster_to_gtiff/CMakeLists.txt

@@ -20,28 +20,44 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 
 set(SAPI_ROOT "" CACHE PATH "Path to the Sandboxed API source tree")
-#   cmake .. -G Ninja -DSAPI_ROOT=$HOME/sapi_root
+set(ENABLE_TESTS OFF CACHE BOOL "Enable GDAL sandbox tests")
+set(GDAL_HEADER_PREFIX "/usr/local/include" CACHE PATH "Prefix of the path to gdal.h")
+set(LIBGDAL_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/lib" CACHE PATH "Prefix of the path to libgdal.a")
+set(LIBPROJ_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/lib" CACHE PATH "Prefix of the path to libproj.a")
 
-set(SAPI_ENABLE_EXAMPLES OFF CACHE BOOL "")
-set(SAPI_ENABLE_TESTS OFF CACHE BOOL "")
 add_subdirectory("${SAPI_ROOT}"
                  "${CMAKE_BINARY_DIR}/sandboxed-api-build"
-                 # Omit this to have the full Sandboxed API in IDE
                  EXCLUDE_FROM_ALL)
 
 add_library(libgdal STATIC IMPORTED)
 set_property(TARGET libgdal PROPERTY IMPORTED_LOCATION 
-            "${CMAKE_CURRENT_SOURCE_DIR}/lib/libgdal.a")
+            "${LIBGDAL_PREFIX}/libgdal.a")
 
+add_library(libproj STATIC IMPORTED)
+set_property(TARGET libproj PROPERTY IMPORTED_LOCATION
+            "${LIBPROJ_PREFIX}/libproj.a")
 
-# TODO: Build PROJ statically and link it as gdal
+message("${LIBGDAL_PREFIX}/libgdal.a")
-# TODO: Use environment variables to specify path prefix to libproj and gdal header
+message("${LIBPROJ_PREFIX}/libproj.a")
-# TODO: Add environment variable to enable tests
+
+# TODO: Try using libgdal-dev and libgdal-proj with SAPI inside Docker or other other isolated environment
+# TODO: Document both install from sources and install from package workflow
+# TODO: Use environment variables to distinguish between those workflows on compile time
 
 target_link_libraries(libgdal INTERFACE
-    crypto expat jpeg 
+    crypto 
-    /usr/local/lib/libproj.so 
+    expat 
-    sqlite3 tiff z pthread m rt dl curl)
+    jpeg
+    libproj
+    sqlite3 
+    tiff 
+    z 
+    pthread 
+    m 
+    rt 
+    dl 
+    curl
+)
 
 add_sapi_library(gdal_sapi
     FUNCTIONS 
@@ -58,7 +74,7 @@ add_sapi_library(gdal_sapi
       GDALRasterIO 
       GDALClose
 
-    INPUTS "../gdal/gdal/gcore/gdal.h"
+    INPUTS "${GDAL_HEADER_PREFIX}/gdal.h"
     LIBRARY libgdal
     LIBRARY_NAME gdal
     
@@ -87,17 +103,18 @@ target_link_libraries(raster_to_gtiff
     sapi::sapi
 )
 
-find_package(GTest REQUIRED)
+if (ENABLE_TESTS)
-include_directories(${GTEST_INCLUDE_DIRS})
+  find_package(GTest REQUIRED)
-
+  include_directories(${GTEST_INCLUDE_DIRS})
-add_executable(tests tests.cc)
+  
-target_link_libraries(tests 
+  add_executable(tests tests.cc)
-  PRIVATE
+  target_link_libraries(tests PRIVATE
-  gdal_sapi
+    gdal_sapi
-  data_retriever
+    data_retriever
-  sapi::sapi
+    sapi::sapi
-  sandbox2::temp_file
+    sandbox2::temp_file
-  sandbox2::fileops
+    sandbox2::fileops
-  ${GTEST_LIBRARIES} 
+    ${GTEST_LIBRARIES} 
-  ${GTEST_MAIN_LIBRARIES}
+    ${GTEST_MAIN_LIBRARIES}
-)
+  )
+endif()

oss-internship-2020/gdal/raster_to_gtiff/README.md

@@ -0,0 +1,21 @@
+# GDAL Raster to GeoTIFF Workflow Sandbox
+This repository is an example of how Sandboxed API can be used with GDAL C Raster API to implement the creation of the GeoTIFF dataset inside the sandbox.
+
+## Workflow details
+Implemented workflow consists of a few steps:
+1. Register needed drivers inside the sandbox
+2. Get specific driver by name (GTiff)
+3. Map output file inside the sandbox and create GeoTIFF dataset backed by this file
+4. Set affine transformation coefficients if needed
+5. Set projection reference string if needed
+6. Write raster bands data to the dataset using RasterIO
+  1. Set No data value if needed
+7. Clean up data and close the dataset
+
+## Implementation details
+
+## Build details
+
+### Build GDAL and PROJ from sources
+
+## Examples

oss-internship-2020/gdal/raster_to_gtiff/gdal_sandbox.h

@@ -15,6 +15,7 @@
 #ifndef GDAL_SANDBOX_H_
 #define GDAL_SANDBOX_H_
 
+#include <iostream>
 #include <string>
 
 #include <syscall.h>
@@ -26,11 +27,17 @@ namespace gdal::sandbox {
 
 class GdalSapiSandbox : public gdalSandbox {
  public:
-  GdalSapiSandbox(std::string path) 
+  GdalSapiSandbox(std::string out_directory_path,
+                  std::string proj_db_path, 
+                  time_t time_limit = 0) 
     : gdalSandbox(),
-      path_(std::move(path)) 
+      out_directory_path_(std::move(out_directory_path)),
-    {}
+      proj_db_path_(std::move(proj_db_path))
+    {
+      SetWallTimeLimit(time_limit).IgnoreError();
+    }
 
+ private:
   std::unique_ptr<sandbox2::Policy> ModifyPolicy(
       sandbox2::PolicyBuilder*) override {
 
@@ -52,14 +59,16 @@ class GdalSapiSandbox : public gdalSandbox {
             __NR_unlink,  // GDALDriver::Delete()
         })
         // TODO: Deal with proj.db so you don't need to specify exact path ih the policy
-        .AddFile("/usr/local/share/proj/proj.db")  // proj.db is required
+        // Add proj path a an environment variable, check it before calling constructor
-        .AddDirectory("/usr/local/lib")  // To add libproj.so.19.1.1
+        // Use default path if there is no variable
-        .AddDirectory(path_, /*is_ro=*/false)
+        // If there is no file on the default path return an error
+        .AddFile(proj_db_path_)  // proj.db is required for some projections
+        .AddDirectory(out_directory_path_, /*is_ro=*/false)
         .BuildOrDie();
-    }
+  }
 
- private:
+  std::string out_directory_path_;
-  std::string path_;
+  std::string proj_db_path_;
 };
 
 }  // namespace gdal::sandbox

oss-internship-2020/gdal/raster_to_gtiff/get_raster_data.cc

@@ -25,26 +25,6 @@ namespace {
 
 inline constexpr int kGeoTransformSize = 6;
 
-void PrintInformationAboutDataset(GDALDatasetH dataset, GDALDriverH driver) {
-  double adfGeoTransform[6];
-  printf("Driver: %s/%s\n",
-      GDALGetDriverShortName(driver),
-      GDALGetDriverLongName(driver));
-  printf("Size is %dx%dx%d\n",
-        GDALGetRasterXSize(dataset),
-        GDALGetRasterYSize(dataset),
-        GDALGetRasterCount(dataset) );
-  if(GDALGetProjectionRef(dataset) != NULL )
-      printf("Projection is `%s'\n", GDALGetProjectionRef(dataset));
-  if(GDALGetGeoTransform(dataset, adfGeoTransform) == CE_None)
-  {
-      printf("Origin = (%.6f,%.6f)\n",
-            adfGeoTransform[0], adfGeoTransform[3]);
-      printf("Pixel Size = (%.6f,%.6f)\n",
-            adfGeoTransform[1], adfGeoTransform[5]);
-  }
-}
-
 }  // namespace
 
 RasterDataset GetRasterBandsFromFile(const std::string& filename) {
@@ -90,7 +70,7 @@ RasterDataset GetRasterBandsFromFile(const std::string& filename) {
     std::vector<int> band_raster_data;
     band_raster_data.resize(width * height);
 
-    // GDALRasterIO with GF_Read should use the same type (GDT_Int32)
+    // GDALRasterIO with GF_Write should use the same type (GDT_Int32)
     GDALRasterIO(band, GF_Read, 0, 0, width, height, band_raster_data.data(),
         width, height, GDT_Int32, 0, 0);
 

oss-internship-2020/gdal/raster_to_gtiff/get_raster_data.h

@@ -26,7 +26,7 @@ struct RasterBandData {
   int height;
   std::vector<int> data;
   int data_type;  // Corresponds to the GDALDataType enum
-  int color_interp;  // Corresponds to the 
+  int color_interp;  // Corresponds to the GDALColorInterp enum
   std::optional<double> no_data_value;
 };
 
@@ -44,4 +44,4 @@ bool operator==(const RasterDataset& lhs, const RasterDataset& rhs);
 
 }  // namespace gdal::sandbox::tests::parser
 
-#endif  // GET_RASTER_DATA_H
+#endif  // GET_RASTER_DATA_H_

oss-internship-2020/gdal/raster_to_gtiff/raster_to_gtiff.cc

@@ -13,9 +13,11 @@
 // limitations under the License.
 
 #include <iostream>
+#include <cstdlib>
 #include <vector>
 #include <string>
 #include <filesystem>
+#include <optional>
 
 #include "sandboxed_api/sandbox2/util/fileops.h"
 
@@ -30,6 +32,22 @@ namespace {
   }
 
 inline constexpr absl::string_view kDriverName = "GTiff";
+inline constexpr absl::string_view kProjDbEnvVariableName = "PROJ_PATH";
+inline constexpr absl::string_view kDefaultProjDbPath 
+    = "/usr/local/share/proj/proj.db";
+
+std::optional<std::string> GetProjDbPath() {
+  const char* proj_db_path_ptr = std::getenv(kProjDbEnvVariableName.data());
+
+  std::string proj_db_path = proj_db_path_ptr == nullptr ? 
+      std::string(kDefaultProjDbPath) : std::string(proj_db_path_ptr);
+
+  if (!std::filesystem::exists(proj_db_path)) {
+    return std::nullopt;
+  }
+
+  return proj_db_path;
+}
 
 absl::Status SaveToGTiff(gdal::sandbox::tests::parser::RasterDataset bands_data, 
     std::string out_file) {
@@ -42,7 +60,11 @@ absl::Status SaveToGTiff(gdal::sandbox::tests::parser::RasterDataset bands_data,
     return absl::FailedPreconditionError("Error getting output file directory");
   }
 
-  GdalSapiSandbox sandbox(output_data_folder);
+  std::optional<std::string> proj_db_path = GetProjDbPath();
+  SAPI_FAIL_IF_NOT(proj_db_path != std::nullopt,
+      "Specified proj.db does not exist");
+
+  GdalSapiSandbox sandbox(output_data_folder, std::move(proj_db_path.value()));
   SAPI_RETURN_IF_ERROR(sandbox.Init());
   gdalApi api(&sandbox);
   SAPI_RETURN_IF_ERROR(api.GDALAllRegister());
@@ -111,7 +133,7 @@ absl::Status SaveToGTiff(gdal::sandbox::tests::parser::RasterDataset bands_data,
   if (bands_data.wkt_projection.length() > 0) {
     sapi::v::ConstCStr wkt_projection_ptr(bands_data.wkt_projection.data());
     SAPI_ASSIGN_OR_RETURN(sapi::StatusOr<CPLErr> result, 
-        api.GDALSetProjection(&dataset_ptr, wkt_projection_ptr.PtrBefore())); 
+        api.GDALSetProjection(&dataset_ptr, wkt_projection_ptr.PtrBefore()));
     SAPI_FAIL_IF_NOT(result.value() == CPLErr::CE_None,
         "Error setting wkt projection");
   }

oss-internship-2020/gdal/raster_to_gtiff/tests.cc

@@ -14,6 +14,7 @@
 
 #include <iostream>
 #include <optional>
+#include <filesystem>
 #include <string>
 
 #include "gtest/gtest.h"
@@ -29,6 +30,9 @@ namespace {
 
 inline constexpr absl::string_view kDriverName = "GTiff";
 inline constexpr absl::string_view kTempFilePrefix = "temp_data";
+inline constexpr absl::string_view kProjDbEnvVariableName = "PROJ_PATH";
+inline constexpr absl::string_view kDefaultProjDbPath 
+    = "/usr/local/share/proj/proj.db";
 inline constexpr absl::string_view kFirstTestDataPath = 
     "../testdata/map.tif";
 inline constexpr absl::string_view kSecondTestDataPath = 
@@ -36,6 +40,19 @@ inline constexpr absl::string_view kSecondTestDataPath =
 inline constexpr absl::string_view kThirdTestDataPath = 
     "../testdata/earth_map.tif";
 
+std::optional<std::string> GetProjDbPath() {
+  const char* proj_db_path_ptr = std::getenv(kProjDbEnvVariableName.data());
+
+  std::string proj_db_path = proj_db_path_ptr == nullptr ? 
+      std::string(kDefaultProjDbPath) : std::string(proj_db_path_ptr);
+
+  if (!std::filesystem::exists(proj_db_path)) {
+    return std::nullopt;
+  }
+
+  return proj_db_path;
+}
+
 // RAII wrapper that creates temporary file and automatically unlinks it
 class TempFile {
  public:
@@ -75,9 +92,11 @@ class RasterToGTiffProcessor : public sapi::Transaction {
  public:
   explicit RasterToGTiffProcessor(std::string out_filename, 
                                   std::string out_path,
-                                  parser::RasterDataset data, 
+                                  std::string proj_db_path,
+                                  parser::RasterDataset data,
                                   int retry_count = 0) 
-    : sapi::Transaction(std::make_unique<GdalSapiSandbox>(out_path)),
+    : sapi::Transaction(std::make_unique<GdalSapiSandbox>(out_path, 
+        std::move(proj_db_path))),
       out_filename_(std::move(out_filename)),
       out_path_(std::move(out_path)),
       data_(std::move(data))
@@ -203,8 +222,14 @@ TEST_P(TestGTiffProcessor, TestProcessorOnGTiffData) {
   parser::RasterDataset original_bands_data = 
       parser::GetRasterBandsFromFile(filename);
 
+  std::optional<std::string> proj_db_path = GetProjDbPath();
+  ASSERT_TRUE(proj_db_path != std::nullopt)
+      << "Specified proj.db does not exist";
+
   RasterToGTiffProcessor processor(tempfile_.GetPath(), 
-      sandbox2::file_util::fileops::GetCWD(), original_bands_data);
+      sandbox2::file_util::fileops::GetCWD(), std::move(proj_db_path.value()),
+      original_bands_data);
+  
   ASSERT_EQ(processor.Run(), absl::OkStatus())
       << "Error creating new GTiff dataset inside sandbox";