Merge pull request #349 from BigVan/main

[feat.] Support uploading 'output' to registry while merging layers

src/overlaybd/lsmt/test/test.cpp

@@ -246,6 +246,12 @@ void test_combo(const IMemoryIndex *indexes[], size_t ni, const SegmentMapping s
         auto ret = memcmp(pm, stdrst, nrst * sizeof(stdrst[0]));
         EXPECT_EQ(ret, 0);
     }
+    LOG_INFO("make RO index of ci");
+    auto ro_idx = ci.make_read_only_index();
+    if (ro_idx->size() == nrst) {
+        auto ret = memcmp(ro_idx->buffer(), stdrst, nrst * sizeof(stdrst[0]));
+        EXPECT_EQ(ret, 0);
+    }
 
     ci.backing_index();
     // delete mi;
@@ -599,20 +605,27 @@ TEST_F(FileTest3, stack_files) {
     LOG_INFO("RO valid data: `", stat.valid_data_size);
     merged = lfs->open(fn_merged, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
     EXPECT_EQ(lower->flatten(merged), 0);
-    cout << "verifying flattened layer of lowers" << endl;
+    // auto localfile = lfs->open(fn_merged, O_RDONLY);
+    struct stat st;
+    merged->fstat(&st);
+    cout << "verifying flattened layer of lowers, st_size: "<<st.st_size << endl;
     verify_file(fn_merged);
     delete merged;
     cout << "generating a RW layer by randwrite()" << endl;
     auto upper = create_file_rw();
     auto file = stack_files(upper, lower, 0, true);
     randwrite(file, FLAGS_nwrites);
     verify_file(file);
-    cout << "verifying flattened layer of stacked layers" << endl;
 
     merged = lfs->open(fn_merged, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
     file->flatten(merged);
+    merged->fstat(&st);
+    cout << "verifying flattened layer of stacked layers, st_size: "<<st.st_size << endl;
+    merged->close();
+
     verify_file(fn_merged);
     delete file;
+    delete merged;
 }
 
 TEST_F(FileTest3, seek_data) {

src/overlaybd/registryfs/registryfs.h

@@ -43,10 +43,14 @@ photon::fs::IFileSystem *new_registryfs_v2(PasswordCB callback,
                                            const char *customized_ua = nullptr);
 
 photon::fs::IFile* new_registry_uploader(photon::fs::IFile *lfile,
-                                         std::string &upload_url,
-                                         std::string &username, std::string &password,
+                                         const std::string &upload_url,
+                                         const std::string &username,
+                                         const std::string &password,
                                          uint64_t timeout,
                                          ssize_t upload_bs = -1,
                                          const char *cert_file = nullptr,
                                          const char *key_file = nullptr);
+
+int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest);
+
 }

src/overlaybd/registryfs/registryfs_v2.cpp

@@ -17,6 +17,7 @@
 #include "registryfs.h"
 
 
+#include <cerrno>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
@@ -561,8 +562,8 @@ class RegistryUploader : public VirtualFile {
     uint64_t m_timeout = -1;
     estring m_token;
 
-    RegistryUploader(IFile *lfile, std::string &upload_url, std::string &username,
-                     std::string &password, uint64_t timeout, ssize_t upload_bs,
+    RegistryUploader(IFile *lfile, const std::string &upload_url, const std::string &username,
+                     const std::string &password, uint64_t timeout, ssize_t upload_bs,
                      photon::net::TLSContext *ctx)
         : m_local_file(lfile), m_origin_upload_url(upload_url), m_username(username), m_password(password),
           m_timeout(timeout), m_tls_ctx(ctx) {
@@ -829,15 +830,15 @@ class RegistryUploader : public VirtualFile {
             LOG_INFO(VALUE(m_upload_url));
             return 0;
         }
-        LOG_ERROR_RETURN(0, -1, "failed to get upload url, code=`", op.status_code);
+        LOG_ERROR_RETURN(0, -1, "failed to get upload url [`], code=`", m_upload_url, op.status_code);
     }
 
 protected:
     photon::net::TLSContext *m_tls_ctx;
 };
 
-IFile *new_registry_uploader(IFile *lfile, std::string &upload_url, std::string &username,
-                             std::string &password, uint64_t timeout, ssize_t upload_bs,
+IFile *new_registry_uploader(IFile *lfile, const std::string &upload_url, const std::string &username,
+                             const std::string &password, uint64_t timeout, ssize_t upload_bs,
                              const char *cert_file, const char *key_file) {
     auto ctx = new_tls_context_from_file(cert_file, key_file);
     if (!ctx) {
@@ -893,3 +894,14 @@ photon::net::TLSContext *new_tls_context_from_file(const char *cert_file, const
     }
     return photon::net::new_tls_context(cert_str.c_str(), key_str.c_str(), nullptr);
 }
+
+int registry_uploader_fini(photon::fs::IFile *uploader, std::string &digest) {
+    if (uploader == nullptr) {
+        LOG_ERRNO_RETURN(0, EINVAL, "invalid pointer");
+    }
+    if (uploader->fsync() < 0) {
+        LOG_ERRNO_RETURN(0, -1, "failed to upload blob");
+    }
+    digest = ((RegistryUploader *) uploader)->m_sha256sum;
+    return 0;
+}

src/tools/CMakeLists.txt

@@ -16,12 +16,12 @@ add_executable(overlaybd-zfile overlaybd-zfile.cpp)
 target_include_directories(overlaybd-zfile PUBLIC ${PHOTON_INCLUDE_DIR})
 target_link_libraries(overlaybd-zfile photon_static overlaybd_lib)
 
-add_executable(overlaybd-apply overlaybd-apply.cpp comm_func.cpp)
+add_executable(overlaybd-apply overlaybd-apply.cpp)
 target_include_directories(overlaybd-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include)
 target_link_libraries(overlaybd-apply photon_static overlaybd_lib overlaybd_image_lib checksum_lib)
 set_target_properties(overlaybd-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib")
 
-add_executable(turboOCI-apply turboOCI-apply.cpp comm_func.cpp)
+add_executable(turboOCI-apply turboOCI-apply.cpp)
 target_include_directories(turboOCI-apply PUBLIC ${PHOTON_INCLUDE_DIR} ${rapidjson_SOURCE_DIR}/include)
 target_link_libraries(turboOCI-apply photon_static overlaybd_lib overlaybd_image_lib)
 set_target_properties(turboOCI-apply PROPERTIES INSTALL_RPATH "/opt/overlaybd/lib")

src/tools/comm_func.cpp

@@ -20,6 +20,7 @@
 #include <photon/fs/subfs.h>
 #include <photon/fs/localfs.h>
 #include <photon/fs/extfs/extfs.h>
+#include "../overlaybd/registryfs/registryfs.h"
 #include "../image_service.h"
 #include "../image_file.h"
 #include "../overlaybd/tar/erofs/liberofs.h"
@@ -91,3 +92,23 @@ photon::fs::IFileSystem *create_erofs_fs(photon::fs::IFile *imgfile, uint64_t bl
 {
     return erofs_create_fs(imgfile, blksz);
 }
+IFile *create_uploader(ZFile::CompressArgs *zfile_args,  IFile *src,
+    const string &upload_url, const string &cred_file_path, uint64_t timeout_minute, uint64_t upload_bs_KB,
+    const string &tls_key_path,  const string &tls_cert_path
+    ){
+
+    zfile_args->overwrite_header = false;
+    LOG_INFO("upload to `", upload_url);
+    std::string username, password;
+    if (load_cred_from_file(cred_file_path, upload_url, username, password) <  0) {
+        fprintf(stderr, "failed to read upload cred file\n");
+        exit(-1);
+    }
+    auto upload_builder = new_registry_uploader(src, upload_url, username, password,
+        timeout_minute*60*1000*1000, upload_bs_KB*1024, tls_cert_path.c_str(), tls_key_path.c_str());
+    if (upload_builder == nullptr) {
+        fprintf(stderr, "failed to init upload\n");
+        exit(-1);
+    }
+    return upload_builder;
+}

src/tools/comm_func.h

@@ -41,6 +41,10 @@ photon::fs::IFile *open_file(const char *fn, int flags, mode_t mode = 0, photon:
 int create_overlaybd(const std::string &srv_config, const std::string &dev_config,
     ImageService *&, photon::fs::IFile *&);
 
+photon::fs::IFile *create_uploader(ZFile::CompressArgs *zfile_args,  IFile *src,
+    const std::string &upload_url, const std::string &cred_file_path, uint64_t timeout_minute, uint64_t upload_bs_KB,
+    const std::string &tls_key_path,  const std::string &tls_cert_path);
+
 photon::fs::IFileSystem *create_ext4fs(photon::fs::IFile *imgfile, bool mkfs,
     bool enable_buffer, const char* root);
 

src/tools/overlaybd-commit.cpp

@@ -21,24 +21,38 @@
 #include "../overlaybd/lsmt/file.h"
 #include "../overlaybd/zfile/zfile.h"
 #include "../overlaybd/tar/tar_file.h"
-#include "../overlaybd/registryfs/registryfs.h"
-#include "../image_service.h"
 #include <errno.h>
 #include <fcntl.h>
 #include <inttypes.h>
-#include <memory>
 #include <string>
 #include <stdio.h>
 #include <stdlib.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include "CLI11.hpp"
+#include "comm_func.h"
+#include "../overlaybd/registryfs/registryfs.h"
+
 
 using namespace std;
 using namespace LSMT;
 using namespace photon::fs;
 
+string commit_msg;
+string uuid, parent_uuid;
+std::string algorithm;
+int block_size = -1;
+std::string data_file_path, index_file_path, commit_file_path, remote_mapping_file;
+bool compress_zfile = false;
+bool build_turboOCI = false;
+bool build_fastoci = false;
+bool tar = false, rm_old = false, seal = false, commit_sealed = false;
+bool verbose = false;
+int compress_threads = 1;
+std::string upload_url, cred_file_path, tls_key_path, tls_cert_path;
+ssize_t upload_bs = 262144;
+
 
 IFile *open_file(IFileSystem *fs, const char *fn, int flags, mode_t mode = 0) {
     auto file = fs->open(fn, flags, mode);
@@ -50,19 +64,6 @@ IFile *open_file(IFileSystem *fs, const char *fn, int flags, mode_t mode = 0) {
 }
 
 int main(int argc, char **argv) {
-    string commit_msg;
-    string uuid, parent_uuid;
-    std::string algorithm;
-    int block_size = -1;
-    std::string data_file_path, index_file_path, commit_file_path, remote_mapping_file;
-    bool compress_zfile = false;
-    bool build_turboOCI = false;
-    bool build_fastoci = false;
-    bool tar = false, rm_old = false, seal = false, commit_sealed = false;
-    bool verbose = false;
-    int compress_threads = 1;
-    std::string upload_url, cred_file_path;
-    ssize_t upload_bs = 262144;
 
     CLI::App app{"this is overlaybd-commit"};
     app.add_option("-m", commit_msg, "add some custom message if needed");
@@ -87,10 +88,18 @@ int main(int argc, char **argv) {
     app.add_option("--upload", upload_url, "registry upload url");
     app.add_option("--upload_bs", upload_bs, "block size for upload, in KB");
     app.add_option("--cred_file_path", cred_file_path, "cred file path for registryfs")->type_name("FILEPATH")->check(CLI::ExistingFile);
+    app.add_option("--tls_key_path", tls_key_path, "TLSKeyPairPath for private Registry")->type_name("FILEPATH")->check(CLI::ExistingFile);
+    app.add_option("--tls_cert_path", tls_cert_path, "TLSCertPath for private Registry")->type_name("FILEPATH")->check(CLI::ExistingFile);
+
+
 
     CLI11_PARSE(app, argc, argv);
     build_turboOCI = build_turboOCI || build_fastoci;
     set_log_output_level(verbose ? 0 : 1);
+    if (tar && (upload_url.empty() == false)){
+        fprintf(stderr, "unsupport option with '-t' and '--upload' at the same time.");
+        exit(-1);
+    }
     photon::init(photon::INIT_EVENT_DEFAULT, photon::INIT_IO_DEFAULT);
     DEFER({photon::fini();});
 
@@ -164,21 +173,10 @@ int main(int argc, char **argv) {
         zfile_args->overwrite_header = true;
 
         if (!upload_url.empty()) {
-            zfile_args->overwrite_header = false;
-            LOG_INFO("upload to `", upload_url);
-            std::string username, password;
-            if (load_cred_from_file(cred_file_path, upload_url, username, password) <  0) {
-                fprintf(stderr, "failed to read upload cred file\n");
-                exit(-1);
-            }
-            upload_builder = new_registry_uploader(out, upload_url, username, password, 2UL*60*1000*1000, upload_bs*1024);
-            if (upload_builder == nullptr) {
-                fprintf(stderr, "failed to init upload\n");
-                exit(-1);
-            }
+            LOG_INFO("enable upload. URL: `, upload_bs: `, tls_key_path: `, tls_cert_path: `", upload_url, upload_bs, tls_key_path, tls_cert_path);
+            upload_builder = create_uploader(zfile_args, out, upload_url, cred_file_path, 2, upload_bs, tls_key_path, tls_cert_path);
             out = upload_builder;
         }
-
         zfile_builder = ZFile::new_zfile_builder(out, zfile_args, false);
         out = zfile_builder;
     } else {
@@ -213,12 +211,12 @@ int main(int argc, char **argv) {
     if (zfile_args) {
         delete zfile_args;
     }
-
-    if (upload_builder != nullptr && upload_builder->fsync() < 0) {
-        fprintf(stderr, "failed to commit or upload");
-        return -1;
-    }
-
+    string digest = "";
+    if (upload_builder != nullptr && registry_uploader_fini(upload_builder, digest) != 0){
+        fprintf(stderr, "failed to commit or upload\n");
+        exit(-1);
+    };
+    fprintf(stderr, "%s\n", digest.c_str());
     delete upload_builder;
     delete fout;
     delete fin;