Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update dependencies #1262

Merged
merged 10 commits into from
Sep 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion artifactory/commands/transferfiles/fulltransfer.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package transferfiles

import (
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"path"
"time"

Expand Down Expand Up @@ -250,7 +251,11 @@ func (m *fullTransferPhase) handleFoundFile(pcWrapper *producerConsumerWrapper,
return
}
// Increment the files count in the directory's node in the snapshot manager, to track its progress.
err = node.IncrementFilesCount(uint64(file.Size))
unsignedFileSize, err := safeconvert.Int64ToUint64(file.Size)
if err != nil {
return fmt.Errorf("failed to convert file size to uint64: %w", err)
}
err = node.IncrementFilesCount(unsignedFileSize)
if err != nil {
return
}
Expand Down
5 changes: 4 additions & 1 deletion artifactory/commands/transferfiles/longpropertycheck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package transferfiles

import (
"encoding/json"
"github.com/jfrog/gofrog/safeconvert"
"io"
"net/http"
"net/http/httptest"
Expand Down Expand Up @@ -56,7 +57,9 @@ func TestProperty(t *testing.T) {
}

func testProperty(t *testing.T, property Property, isLong bool) {
assert.Len(t, property.Value, int(property.valueLength()))
signedValueLen, err := safeconvert.UintToInt(property.valueLength())
assert.NoError(t, err)
assert.Len(t, property.Value, signedValueLen)
long := isLongProperty(property)
assert.Equal(t, isLong, long)
}
Expand Down
2 changes: 1 addition & 1 deletion artifactory/commands/transferfiles/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@ func updateProgress(phase *phaseBase, timeEstMng *state.TimeEstimationManager,
}

if timeEstMng != nil {
timeEstMng.AddChunkStatus(chunk, time.Since(chunkSentTime).Milliseconds())
return timeEstMng.AddChunkStatus(chunk, time.Since(chunkSentTime).Milliseconds())
}
return nil
}
Expand Down
14 changes: 12 additions & 2 deletions artifactory/commands/transferfiles/state/statemanager.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package state

import (
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"path/filepath"
"time"

Expand Down Expand Up @@ -94,7 +96,11 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
log.Info("Calculated transferred files from previous run:", transferredFiles)
log.Info("Calculated transferred bytes from previous run:", transferredSizeBytes)
transferState.CurrentRepo.Phase1Info.TransferredUnits = int64(transferredFiles)
transferState.CurrentRepo.Phase1Info.TransferredSizeBytes = int64(transferredSizeBytes)
signedTransferredSizeBytes, err := safeconvert.Uint64ToInt64(transferredSizeBytes)
if err != nil {
return fmt.Errorf("failed to set transferred size bytes: %w", err)
}
transferState.CurrentRepo.Phase1Info.TransferredSizeBytes = signedTransferredSizeBytes
}

ts.TransferState = transferState
Expand All @@ -110,7 +116,11 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
transferRunStatus.VisitedFolders = 0

transferRunStatus.OverallTransfer.TransferredUnits += int64(transferredFiles)
transferRunStatus.OverallTransfer.TransferredSizeBytes += int64(transferredSizeBytes)
signedTransferredSizeBytes, err := safeconvert.Uint64ToInt64(transferredSizeBytes)
if err != nil {
return fmt.Errorf("failed to set transferred size bytes: %w", err)
}
transferRunStatus.OverallTransfer.TransferredSizeBytes += signedTransferredSizeBytes
return nil
})
}
Expand Down
21 changes: 16 additions & 5 deletions artifactory/commands/transferfiles/state/statemanager_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package state

import (
"github.com/jfrog/gofrog/safeconvert"
"sync"
"testing"
"time"
Expand Down Expand Up @@ -340,9 +341,15 @@ func TestStateConcurrency(t *testing.T) {
assert.Equal(t, 1000, int(stateManager.CurrentRepo.Phase3Info.TransferredSizeBytes))
assert.Equal(t, 1000, int(stateManager.CurrentRepo.Phase3Info.TransferredUnits))
assert.Equal(t, 1000, int(stateManager.OverallTransfer.TransferredSizeBytes))
assert.Equal(t, 1000, int(stateManager.VisitedFolders))
assert.Equal(t, 1000, int(stateManager.DelayedFiles))
assert.Equal(t, 1000, int(stateManager.TransferFailures))
signedVisitedFolders, err := safeconvert.Uint64ToInt(stateManager.VisitedFolders)
assert.NoError(t, err)
assert.Equal(t, 1000, signedVisitedFolders)
signedDelayedFiles, err := safeconvert.Uint64ToInt(stateManager.DelayedFiles)
assert.NoError(t, err)
assert.Equal(t, 1000, signedDelayedFiles)
signedTransferFailures, err := safeconvert.Uint64ToInt(stateManager.TransferFailures)
assert.NoError(t, err)
assert.Equal(t, 1000, signedTransferFailures)

// Concurrently decrement delayed artifacts and transfer failures
for i := 0; i < 500; i++ {
Expand All @@ -356,6 +363,10 @@ func TestStateConcurrency(t *testing.T) {
wg.Wait()

// Assert 500 in delayed artifacts and transfer failures
assert.Equal(t, 500, int(stateManager.DelayedFiles))
assert.Equal(t, 500, int(stateManager.TransferFailures))
signedDelayedFiles, err = safeconvert.Uint64ToInt(stateManager.DelayedFiles)
assert.NoError(t, err)
assert.Equal(t, 500, signedDelayedFiles)
signedTransferFailures, err = safeconvert.Uint64ToInt(stateManager.TransferFailures)
assert.NoError(t, err)
assert.Equal(t, 500, signedTransferFailures)
}
51 changes: 33 additions & 18 deletions artifactory/commands/transferfiles/state/timeestimation.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package state

import (
"errors"
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"time"

"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
Expand Down Expand Up @@ -39,19 +41,23 @@ type TimeEstimationManager struct {
stateManager *TransferStateManager
}

func (tem *TimeEstimationManager) AddChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) {
func (tem *TimeEstimationManager) AddChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) error {
if durationMillis == 0 {
return
return nil
}

tem.addDataChunkStatus(chunkStatus, durationMillis)
return tem.addDataChunkStatus(chunkStatus, durationMillis)
}

func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) {
func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) error {
var chunkSizeBytes int64
for _, file := range chunkStatus.Files {
if file.Status != api.Fail {
tem.CurrentTotalTransferredBytes += uint64(file.SizeBytes)
unsignedSizeBytes, err := safeconvert.Int64ToUint64(file.SizeBytes)
if err != nil {
return fmt.Errorf("failed to calculate the estimated remaining time: %w", err)
}
tem.CurrentTotalTransferredBytes += unsignedSizeBytes
}
if (file.Status == api.Success || file.Status == api.SkippedLargeProps) && !file.ChecksumDeployed {
chunkSizeBytes += file.SizeBytes
Expand All @@ -60,13 +66,13 @@ func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus

// If no files were uploaded regularly (with no errors and not checksum-deployed), don't use this chunk for the time estimation calculation.
if chunkSizeBytes == 0 {
return
return nil
}

workingThreads, err := tem.stateManager.GetWorkingThreads()
if err != nil {
log.Error("Couldn't calculate time estimation:", err.Error())
return
return err
}
speed := calculateChunkSpeed(workingThreads, chunkSizeBytes, durationMillis)
tem.LastSpeeds = append(tem.LastSpeeds, speed)
Expand All @@ -79,10 +85,11 @@ func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus
}
if len(tem.LastSpeeds) == 0 {
tem.SpeedsAverage = 0
return
return err
}
// Calculate speed in bytes/ms
tem.SpeedsAverage = tem.LastSpeedsSum / float64(len(tem.LastSpeeds))
return nil
}

func calculateChunkSpeed(workingThreads int, chunkSizeSum, chunkDuration int64) float64 {
Expand All @@ -108,31 +115,39 @@ func (tem *TimeEstimationManager) GetSpeedString() string {
// 1. 5 minutes not passed since the beginning of the transfer
// 2. No files transferred
// 3. The transfer speed is less than 1 byte per second
func (tem *TimeEstimationManager) GetEstimatedRemainingTimeString() string {
remainingTimeSec := tem.getEstimatedRemainingSeconds()
if remainingTimeSec == 0 {
return "Not available yet"
func (tem *TimeEstimationManager) GetEstimatedRemainingTimeString() (string, error) {
remainingTimeSec, err := tem.getEstimatedRemainingSeconds()
if remainingTimeSec == 0 || err != nil {
return "Not available yet", err
}

return SecondsToLiteralTime(int64(remainingTimeSec), "About ")
signedRemainingTimeSec, err := safeconvert.Uint64ToInt64(remainingTimeSec)
if err != nil {
return "", errors.New("failed to calculate the estimated remaining time: " + err.Error())
}
return SecondsToLiteralTime(signedRemainingTimeSec, "About "), nil
}

func (tem *TimeEstimationManager) getEstimatedRemainingSeconds() uint64 {
func (tem *TimeEstimationManager) getEstimatedRemainingSeconds() (uint64, error) {
if tem.CurrentTotalTransferredBytes == 0 {
// No files transferred
return 0
return 0, nil
}
duration := time.Since(tem.stateManager.startTimestamp)
if duration < minTransferTimeToShowEstimation {
// 5 minutes not yet passed
return 0
return 0, nil
}

transferredBytesInSeconds := tem.CurrentTotalTransferredBytes / uint64(duration.Seconds())
if transferredBytesInSeconds == 0 {
// Less than 1 byte per second
return 0
return 0, nil
}
remainingBytes := tem.stateManager.OverallTransfer.TotalSizeBytes - tem.stateManager.OverallTransfer.TransferredSizeBytes
return uint64(remainingBytes) / transferredBytesInSeconds
unsignedRemainingBytes, err := safeconvert.Int64ToUint64(remainingBytes)
if err != nil {
return 0, fmt.Errorf("failed to calculate the estimated remaining time: %w", err)
}
return unsignedRemainingBytes / transferredBytesInSeconds, nil
}
40 changes: 29 additions & 11 deletions artifactory/commands/transferfiles/state/timeestimation_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package state

import (
"github.com/jfrog/gofrog/safeconvert"
"testing"
"time"

Expand Down Expand Up @@ -51,7 +52,9 @@ func TestGetSpeed(t *testing.T) {
addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
assert.Equal(t, 7.5, timeEstMng.getSpeed())
assert.Equal(t, "7.500 MB/s", timeEstMng.GetSpeedString())
assert.NotZero(t, timeEstMng.getEstimatedRemainingSeconds())
estimatedRemainingSeconds, err := timeEstMng.getEstimatedRemainingSeconds()
assert.NoError(t, err)
assert.NotZero(t, estimatedRemainingSeconds)

// Chunk 2: the upload of one of the files failed and the files are not included in the repository's total size (includedInTotalSize == false)
chunkStatus2 := api.ChunkStatus{
Expand All @@ -63,20 +66,27 @@ func TestGetSpeed(t *testing.T) {
addChunkStatus(t, timeEstMng, chunkStatus2, 2, false, 5*milliSecsInSecond)
assert.Equal(t, float64(8), timeEstMng.getSpeed())
assert.Equal(t, "8.000 MB/s", timeEstMng.GetSpeedString())
assert.NotZero(t, timeEstMng.getEstimatedRemainingSeconds())
estimatedRemainingSeconds, err = timeEstMng.getEstimatedRemainingSeconds()
assert.NoError(t, err)
assert.NotZero(t, estimatedRemainingSeconds)
}

func TestGetEstimatedRemainingSeconds(t *testing.T) {
timeEstMng, cleanUp := initTimeEstimationDataTest(t)
defer cleanUp()

timeEstMng.CurrentTotalTransferredBytes = uint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes)
unsignedTotalSizeBytes, err := safeconvert.Int64ToUint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes)
assert.NoError(t, err)
timeEstMng.CurrentTotalTransferredBytes = unsignedTotalSizeBytes
timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = timeEstMng.stateManager.OverallTransfer.TotalSizeBytes
assert.Zero(t, timeEstMng.getEstimatedRemainingSeconds())
estimatedRemainingSeconds, err := timeEstMng.getEstimatedRemainingSeconds()
assert.NoError(t, err)
assert.Zero(t, estimatedRemainingSeconds)

timeEstMng.CurrentTotalTransferredBytes = uint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) / 2
timeEstMng.CurrentTotalTransferredBytes = unsignedTotalSizeBytes / 2
timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = timeEstMng.stateManager.OverallTransfer.TotalSizeBytes / 2
calculatedEstimatedSeconds := timeEstMng.getEstimatedRemainingSeconds()
calculatedEstimatedSeconds, err := timeEstMng.getEstimatedRemainingSeconds()
assert.NoError(t, err)
assert.NotZero(t, calculatedEstimatedSeconds)
}

Expand All @@ -90,7 +100,9 @@ func TestGetEstimatedRemainingTimeStringNotAvailableYet(t *testing.T) {
createFileUploadStatusResponse(repo1Key, 8*rtServicesUtils.SizeMiB, true, api.Success),
},
}
assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString())
estimatedRemainingTime, err := timeEstMng.GetEstimatedRemainingTimeString()
assert.NoError(t, err)
assert.Equal(t, "Not available yet", estimatedRemainingTime)
addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
assert.Equal(t, "Not available yet", timeEstMng.GetSpeedString())
}
Expand All @@ -101,17 +113,23 @@ func TestGetEstimatedRemainingTimeString(t *testing.T) {

// Test "Not available yet" by setting the TotalTransferredBytes to 0
timeEstMng.CurrentTotalTransferredBytes = 0
assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString())
estimatedRemainingTime, err := timeEstMng.GetEstimatedRemainingTimeString()
assert.NoError(t, err)
assert.Equal(t, "Not available yet", estimatedRemainingTime)

// Test "About 1 minute" by setting the transferred bytes to 80%
timeEstMng.CurrentTotalTransferredBytes = uint64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.8)
timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = int64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.8)
assert.Equal(t, "About 1 minute", timeEstMng.GetEstimatedRemainingTimeString())
estimatedRemainingTime, err = timeEstMng.GetEstimatedRemainingTimeString()
assert.NoError(t, err)
assert.Equal(t, "About 1 minute", estimatedRemainingTime)

// Test "Less than a minute" by setting the transferred bytes to 90%
timeEstMng.CurrentTotalTransferredBytes = uint64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.9)
timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = int64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.9)
assert.Equal(t, "Less than a minute", timeEstMng.GetEstimatedRemainingTimeString())
estimatedRemainingTime, err = timeEstMng.GetEstimatedRemainingTimeString()
assert.NoError(t, err)
assert.Equal(t, "Less than a minute", estimatedRemainingTime)
}

func newDefaultTimeEstimationManager(t *testing.T, buildInfoRepos bool) *TimeEstimationManager {
Expand Down Expand Up @@ -220,7 +238,7 @@ func addChunkStatus(t *testing.T, timeEstMng *TimeEstimationManager, chunkStatus
assert.NoError(t, err)
}
assert.NoError(t, timeEstMng.stateManager.SetWorkingThreads(workingThreads))
timeEstMng.AddChunkStatus(chunkStatus, durationMillis)
assert.NoError(t, timeEstMng.AddChunkStatus(chunkStatus, durationMillis))
}

func assertTransferredSizes(t *testing.T, stateManager *TransferStateManager, repo1expected, repo2expected int64) {
Expand Down
13 changes: 10 additions & 3 deletions artifactory/commands/transferfiles/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ func ShowStatus() error {
return nil
}

addOverallStatus(stateManager, &output, stateManager.GetRunningTimeString())
if err = addOverallStatus(stateManager, &output, stateManager.GetRunningTimeString()); err != nil {
return err
}
if stateManager.CurrentRepoKey != "" {
transferState, exists, err := state.LoadTransferState(stateManager.CurrentRepoKey, false)
if err != nil {
Expand All @@ -71,20 +73,25 @@ func isStopping() (bool, error) {
return fileutils.IsFileExists(filepath.Join(transferDir, StopFileName), false)
}

func addOverallStatus(stateManager *state.TransferStateManager, output *strings.Builder, runningTime string) {
func addOverallStatus(stateManager *state.TransferStateManager, output *strings.Builder, runningTime string) error {
addTitle(output, "Overall Transfer Status")
addString(output, coreutils.RemoveEmojisIfNonSupportedTerminal("🟢"), "Status", "Running", 3)
addString(output, "🏃", "Running for", runningTime, 3)
addString(output, "🗄 ", "Storage", sizeToString(stateManager.OverallTransfer.TransferredSizeBytes)+" / "+sizeToString(stateManager.OverallTransfer.TotalSizeBytes)+calcPercentageInt64(stateManager.OverallTransfer.TransferredSizeBytes, stateManager.OverallTransfer.TotalSizeBytes), 3)
addString(output, "📦", "Repositories", fmt.Sprintf("%d / %d", stateManager.TotalRepositories.TransferredUnits, stateManager.TotalRepositories.TotalUnits)+calcPercentageInt64(stateManager.TotalRepositories.TransferredUnits, stateManager.TotalRepositories.TotalUnits), 2)
addString(output, "🧵", "Working threads", strconv.Itoa(stateManager.WorkingThreads), 2)
addString(output, "⚡", "Transfer speed", stateManager.GetSpeedString(), 2)
addString(output, "⌛", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1)
estimatedRemainingTime, err := stateManager.GetEstimatedRemainingTimeString()
if err != nil {
return err
}
addString(output, "⌛", "Estimated time remaining", estimatedRemainingTime, 1)
failureTxt := strconv.FormatUint(stateManager.TransferFailures, 10)
if stateManager.TransferFailures > 0 {
failureTxt += " (" + progressbar.RetryFailureContentNote + ")"
}
addString(output, "❌", "Transfer failures", failureTxt, 2)
return nil
}

func calcPercentageInt64(transferred, total int64) string {
Expand Down
Loading
Loading