Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support new file storage for Add with ModSpec #521

Merged
merged 1 commit into from
Nov 5, 2024
Merged

feat: support new file storage for Add with ModSpec #521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions pkg/client/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"kcl-lang.io/kcl-go/pkg/kcl"
"kcl-lang.io/kpm/pkg/downloader"
"kcl-lang.io/kpm/pkg/env"
"kcl-lang.io/kpm/pkg/features"
"kcl-lang.io/kpm/pkg/opt"
pkg "kcl-lang.io/kpm/pkg/package"
"kcl-lang.io/kpm/pkg/reporter"
Expand Down Expand Up @@ -82,6 +83,18 @@ func TestWithGlobalLock(t *testing.T) {
test.RunTestWithGlobalLock(t, "TestDownloadGitWithPackage", testDownloadGitWithPackage)
test.RunTestWithGlobalLock(t, "TestModandLockFilesWithGitPackageDownload", testModandLockFilesWithGitPackageDownload)
test.RunTestWithGlobalLock(t, "TestDependencyGraph", testDependencyGraph)
test.RunTestWithGlobalLock(t, "TestAddWithModSpec", testAddWithModSpec)
test.RunTestWithGlobalLock(t, "TestRunRemoteWithArgsInvalid", testRunRemoteWithArgsInvalid)
test.RunTestWithGlobalLock(t, "TestRunRemoteWithArgs", testRunRemoteWithArgs)
test.RunTestWithGlobalLock(t, "TestRunWithNoSumCheck", testRunWithGitPackage)
test.RunTestWithGlobalLock(t, "TestRunGit", testRunGit)
test.RunTestWithGlobalLock(t, "TestRunOciWithSettingsFile", testRunOciWithSettingsFile)
test.RunTestWithGlobalLock(t, "TestVendorWithGlobalLock", testVendorWithGlobalLock)
test.RunTestWithGlobalLock(t, "TestPull", testPull)
test.RunTestWithGlobalLock(t, "TestPullWithInsecureSkipTLSverify", testPullWithInsecureSkipTLSverify)
test.RunTestWithGlobalLock(t, "TestPullWithModSpec", testPullWithModSpec)

features.Enable(features.SupportNewStorage)
test.RunTestWithGlobalLock(t, "testAddWithModSpec", testAddWithModSpec)
}

Expand Down Expand Up @@ -2004,7 +2017,7 @@ func TestRunLocalWithArgs(t *testing.T) {
}
}

func TestRunRemoteWithArgsInvalid(t *testing.T) {
func testRunRemoteWithArgsInvalid(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

Expand Down Expand Up @@ -2033,7 +2046,7 @@ func TestRunRemoteWithArgsInvalid(t *testing.T) {
}
}

func TestRunRemoteWithArgs(t *testing.T) {
func testRunRemoteWithArgs(t *testing.T) {
pkgPath := getTestDir("test_run_options")
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)
Expand Down
6 changes: 3 additions & 3 deletions pkg/client/deperated_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func TestRunWithNoSumCheck(t *testing.T) {
}()
}

func TestRunWithGitPackage(t *testing.T) {
func testRunWithGitPackage(t *testing.T) {
pkgPath := getTestDir("test_run_git_package")

kpmcli, err := NewKpmClient()
Expand Down Expand Up @@ -84,7 +84,7 @@ func testRunWithOciDownloader(t *testing.T) {
assert.Equal(t, res.GetRawYamlResult(), "The_first_kcl_program: Hello World!")
}

func TestRunGit(t *testing.T) {
func testRunGit(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)

Expand All @@ -107,7 +107,7 @@ func TestRunGit(t *testing.T) {
assert.Equal(t, utils.RmNewline(string(bytes)), utils.RmNewline(string(resultStr)))
}

func TestRunOciWithSettingsFile(t *testing.T) {
func testRunOciWithSettingsFile(t *testing.T) {
kpmcli, err := NewKpmClient()
assert.Equal(t, err, nil)
kpmcli.SetLogWriter(nil)
Expand Down
6 changes: 3 additions & 3 deletions pkg/client/pull_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"kcl-lang.io/kpm/pkg/downloader"
)

func TestPull(t *testing.T) {
func testPull(t *testing.T) {
pulledPath := getTestDir("test_pull")
defer func() {
err := os.RemoveAll(filepath.Join(pulledPath, "oci"))
Expand Down Expand Up @@ -62,7 +62,7 @@ func TestPull(t *testing.T) {
}()
}

func TestPullWithInsecureSkipTLSverify(t *testing.T) {
func testPullWithInsecureSkipTLSverify(t *testing.T) {
pulledPath := getTestDir("test_pull")

kpmcli, err := NewKpmClient()
Expand Down Expand Up @@ -128,7 +128,7 @@ func TestInsecureSkipTLSverifyOCIRegistry(t *testing.T) {
assert.Equal(t, buf.String(), "Called Success\n")
}

func TestPullWithModSpec(t *testing.T) {
func testPullWithModSpec(t *testing.T) {
pulledPath := getTestDir("test_pull_with_modspec")
defer func() {
err := os.RemoveAll(filepath.Join(pulledPath, "oci"))
Expand Down
2 changes: 1 addition & 1 deletion pkg/client/vendor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func testVendorWithMVS(t *testing.T) {
assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "vendor", "helloworld_0.1.1")), false)
}

func TestVendorWithGlobalLock(t *testing.T) {
func testVendorWithGlobalLock(t *testing.T) {
test.RunTestWithGlobalLock(t, "TestVendorDeps", testVendorDeps)
test.RunTestWithGlobalLock(t, "TestVendorWithMVS", testVendorWithMVS)
}
219 changes: 133 additions & 86 deletions pkg/downloader/downloader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"os"
"path/filepath"

gogit "github.com/go-git/go-git/v5"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can refer to fluxcd's best practices for handling Git dependencies, which involve whether to use libgit2 and whether CGO is enabled.

v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/otiai10/copy"
"kcl-lang.io/kpm/pkg/constants"
Expand Down Expand Up @@ -139,34 +140,68 @@ func (d *GitDownloader) LatestVersion(opts *DownloadOptions) (string, error) {
// TODO：supports fetch the latest commit from the git bare repo,
// after totally transfer to the new storage.
// refer to cargo: https://github.com/rust-lang/cargo/blob/3dedb85a25604bdbbb8d3bf4b03162961a4facd0/crates/cargo-util-schemas/src/core/source_kind.rs#L133
var err error
tmp, err := os.MkdirTemp("", "")
if err != nil {
return "", err
}
tmp = filepath.Join(tmp, constants.GitScheme)

defer func() {
err = os.RemoveAll(tmp)
}()

repo, err := git.CloneWithOpts(
git.WithCommit(opts.Source.Commit),
git.WithBranch(opts.Source.Branch),
git.WithTag(opts.Source.Git.Tag),
git.WithRepoURL(opts.Source.Git.Url),
git.WithLocalPath(tmp),
)
var repo *gogit.Repository
if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
var err error
tmp, err := os.MkdirTemp("", "")
if err != nil {
return "", err
}
tmp = filepath.Join(tmp, constants.GitScheme)

defer func() {
err = os.RemoveAll(tmp)
}()

repo, err = git.CloneWithOpts(
git.WithCommit(opts.Source.Commit),
git.WithBranch(opts.Source.Branch),
git.WithTag(opts.Source.Git.Tag),
git.WithRepoURL(opts.Source.Git.Url),
git.WithLocalPath(tmp),
)

if err != nil {
return "", err
}
if err != nil {
return "", err
}
} else {
// Get the latest commit from the git repository cache
cacheFullPath := opts.CachePath
// If the cache bare git repository exists, fetch the latest commit from the cache.
if git.IsGitBareRepo(cacheFullPath) {
err := git.Fetch(cacheFullPath)
if err != nil {
return "", err
}
repo, err = gogit.PlainOpen(cacheFullPath)
if err != nil {
return "", err
}
} else {
// If not, clone the bare repository from the remote git repository, update the cache.
cloneOpts := []git.CloneOption{
git.WithCommit(opts.Source.Git.Commit),
git.WithBranch(opts.Source.Git.Branch),
git.WithTag(opts.Source.Git.Tag),
}

repo, err = git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(opts.Source.Git.Url),
git.WithLocalPath(cacheFullPath),
git.WithBare(true),
)...,
)
if err != nil {
return "", err
}
}
}
ref, err := repo.Head()
if err != nil {
return "", err
}

commit, err := repo.CommitObject(ref.Hash())
if err != nil {
return "", err
Expand Down Expand Up @@ -240,7 +275,6 @@ func (d *DepDownloader) Download(opts *DownloadOptions) error {
localPath := opts.LocalPath
cacheFullPath := opts.CachePath
if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
cacheFullPath = filepath.Join(opts.CachePath, opts.Source.LocalPath())
if utils.DirExists(cacheFullPath) && utils.DirExists(filepath.Join(cacheFullPath, constants.KCL_MOD)) &&
// If the version in modspec is empty, meanings the latest version is needed.
// The latest version should be requested first and the cache should be updated.
Expand All @@ -261,49 +295,56 @@ func (d *DepDownloader) Download(opts *DownloadOptions) error {
}
}

opts.LocalPath = tmpDir
// Dispatch the download to the specific downloader by package source.
if opts.Source.Oci != nil {
if d.OciDownloader == nil {
d.OciDownloader = &OciDownloader{}
}
err := d.OciDownloader.Download(opts)
if err != nil {
return err
// If the dependency package is already exist,
// Skip the download process.
if utils.DirExists(localPath) &&
utils.DirExists(filepath.Join(localPath, constants.KCL_MOD)) {
return nil
} else {
opts.LocalPath = tmpDir
// Dispatch the download to the specific downloader by package source.
if opts.Source.Oci != nil {
if d.OciDownloader == nil {
d.OciDownloader = &OciDownloader{}
}
err := d.OciDownloader.Download(opts)
if err != nil {
return err
}
}
}

if opts.Source.Git != nil {
if d.GitDownloader == nil {
d.GitDownloader = &GitDownloader{}
if opts.Source.Git != nil {
if d.GitDownloader == nil {
d.GitDownloader = &GitDownloader{}
}
err := d.GitDownloader.Download(opts)
if err != nil {
return err
}
}
err := d.GitDownloader.Download(opts)
if err != nil {
return err

// rename the tmp dir to the local path.
if utils.DirExists(localPath) {
err := os.RemoveAll(localPath)
if err != nil {
return err
}
}
}

// rename the tmp dir to the local path.
if utils.DirExists(localPath) {
err := os.RemoveAll(localPath)
// Move the downloaded package to the local path.
// On unix, after the move, the tmp dir will be removed.
err = utils.MoveOrCopy(tmpDir, localPath)
if err != nil {
return err
}
}

// Move the downloaded package to the local path.
// On unix, after the move, the tmp dir will be removed.
err = utils.MoveOrCopy(tmpDir, localPath)
if err != nil {
return err
}

if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
// Enable the cache, update the dependency package to the cache path.
if cacheFullPath != localPath {
err := copy.Copy(localPath, cacheFullPath)
if err != nil {
return err
if ok, err := features.Enabled(features.SupportNewStorage); err == nil && !ok && opts.EnableCache {
// Enable the cache, update the dependency package to the cache path.
if cacheFullPath != localPath {
err := copy.Copy(localPath, cacheFullPath)
if err != nil {
return err
}
}
}
}
Expand Down Expand Up @@ -368,12 +409,8 @@ func (d *OciDownloader) Download(opts *DownloadOptions) error {

if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok {
if opts.EnableCache {
hash, err := ociSource.Hash()
if err != nil {
return err
}
cacheFullPath := filepath.Join(opts.CachePath, hash)
localFullPath := filepath.Join(opts.LocalPath, hash)
cacheFullPath := opts.CachePath
localFullPath := opts.LocalPath

if utils.DirExists(localFullPath) &&
utils.DirExists(filepath.Join(localFullPath, constants.KCL_MOD)) {
Expand Down Expand Up @@ -463,13 +500,8 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {

if ok, err := features.Enabled(features.SupportNewStorage); err == nil && ok {
if opts.EnableCache {
hash, err := gitSource.Hash()
if err != nil {
return err
}

cacheFullPath := filepath.Join(opts.CachePath, hash)
localFullPath := filepath.Join(opts.LocalPath, hash)
cacheFullPath := opts.CachePath
localFullPath := opts.LocalPath
// Check if the package is already downloaded, if so, skip the download.
if utils.DirExists(localFullPath) &&
utils.DirExists(filepath.Join(localFullPath, constants.KCL_MOD)) {
Expand All @@ -486,30 +518,45 @@ func (d *GitDownloader) Download(opts *DownloadOptions) error {
// If failed to clone the bare repository from the cache path,
// clone the bare repository from the remote git repository, update the cache.
if err != nil {
_, err := git.CloneWithOpts(
// If the bare repository cache exists, fetch the latest commit from the cache.
if utils.DirExists(cacheFullPath) && git.IsGitBareRepo(cacheFullPath) {
err := git.Fetch(cacheFullPath)
if err != nil {
return err
}
} else {
// If not, clone the bare repository from the remote git repository, update the cache.
if utils.DirExists(cacheFullPath) {
err = os.Remove(cacheFullPath)
if err != nil {
return err
}
}
_, err := git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(gitSource.Url),
git.WithLocalPath(cacheFullPath),
git.WithBare(true),
)...,
)
if err != nil {
return err
}
}
// After cloning the bare repository,
// Clone the repository from the cache path to the local path.
_, err = git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(gitSource.Url),
git.WithLocalPath(cacheFullPath),
git.WithBare(true),
git.WithRepoURL(cacheFullPath),
git.WithLocalPath(localFullPath),
)...,
)
if err != nil {
return err
}
}
// After cloning the bare repository,
// Clone the repository from the cache path to the local path.
_, err = git.CloneWithOpts(
append(
cloneOpts,
git.WithRepoURL(cacheFullPath),
git.WithLocalPath(localFullPath),
)...,
)
if err != nil {
return err
}
}
}
} else {
Expand Down
Loading