Skip to content

Commit

Permalink
Update dependencies (#1262)
Browse files Browse the repository at this point in the history
  • Loading branch information
omerzi authored Sep 9, 2024
1 parent 5e23772 commit c92e0ab
Show file tree
Hide file tree
Showing 18 changed files with 193 additions and 68 deletions.
7 changes: 6 additions & 1 deletion artifactory/commands/transferfiles/fulltransfer.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package transferfiles

import (
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"path"
"time"

Expand Down Expand Up @@ -250,7 +251,11 @@ func (m *fullTransferPhase) handleFoundFile(pcWrapper *producerConsumerWrapper,
return
}
// Increment the files count in the directory's node in the snapshot manager, to track its progress.
err = node.IncrementFilesCount(uint64(file.Size))
unsignedFileSize, err := safeconvert.Int64ToUint64(file.Size)
if err != nil {
return fmt.Errorf("failed to convert file size to uint64: %w", err)
}
err = node.IncrementFilesCount(unsignedFileSize)
if err != nil {
return
}
Expand Down
5 changes: 4 additions & 1 deletion artifactory/commands/transferfiles/longpropertycheck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package transferfiles

import (
"encoding/json"
"github.com/jfrog/gofrog/safeconvert"
"io"
"net/http"
"net/http/httptest"
Expand Down Expand Up @@ -56,7 +57,9 @@ func TestProperty(t *testing.T) {
}

func testProperty(t *testing.T, property Property, isLong bool) {
assert.Len(t, property.Value, int(property.valueLength()))
signedValueLen, err := safeconvert.UintToInt(property.valueLength())
assert.NoError(t, err)
assert.Len(t, property.Value, signedValueLen)
long := isLongProperty(property)
assert.Equal(t, isLong, long)
}
Expand Down
2 changes: 1 addition & 1 deletion artifactory/commands/transferfiles/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@ func updateProgress(phase *phaseBase, timeEstMng *state.TimeEstimationManager,
}

if timeEstMng != nil {
timeEstMng.AddChunkStatus(chunk, time.Since(chunkSentTime).Milliseconds())
return timeEstMng.AddChunkStatus(chunk, time.Since(chunkSentTime).Milliseconds())
}
return nil
}
Expand Down
14 changes: 12 additions & 2 deletions artifactory/commands/transferfiles/state/statemanager.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package state

import (
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"path/filepath"
"time"

Expand Down Expand Up @@ -94,7 +96,11 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
log.Info("Calculated transferred files from previous run:", transferredFiles)
log.Info("Calculated transferred bytes from previous run:", transferredSizeBytes)
transferState.CurrentRepo.Phase1Info.TransferredUnits = int64(transferredFiles)
transferState.CurrentRepo.Phase1Info.TransferredSizeBytes = int64(transferredSizeBytes)
signedTransferredSizeBytes, err := safeconvert.Uint64ToInt64(transferredSizeBytes)
if err != nil {
return fmt.Errorf("failed to set transferred size bytes: %w", err)
}
transferState.CurrentRepo.Phase1Info.TransferredSizeBytes = signedTransferredSizeBytes
}

ts.TransferState = transferState
Expand All @@ -110,7 +116,11 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
transferRunStatus.VisitedFolders = 0

transferRunStatus.OverallTransfer.TransferredUnits += int64(transferredFiles)
transferRunStatus.OverallTransfer.TransferredSizeBytes += int64(transferredSizeBytes)
signedTransferredSizeBytes, err := safeconvert.Uint64ToInt64(transferredSizeBytes)
if err != nil {
return fmt.Errorf("failed to set transferred size bytes: %w", err)
}
transferRunStatus.OverallTransfer.TransferredSizeBytes += signedTransferredSizeBytes
return nil
})
}
Expand Down
21 changes: 16 additions & 5 deletions artifactory/commands/transferfiles/state/statemanager_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package state

import (
"github.com/jfrog/gofrog/safeconvert"
"sync"
"testing"
"time"
Expand Down Expand Up @@ -340,9 +341,15 @@ func TestStateConcurrency(t *testing.T) {
assert.Equal(t, 1000, int(stateManager.CurrentRepo.Phase3Info.TransferredSizeBytes))
assert.Equal(t, 1000, int(stateManager.CurrentRepo.Phase3Info.TransferredUnits))
assert.Equal(t, 1000, int(stateManager.OverallTransfer.TransferredSizeBytes))
assert.Equal(t, 1000, int(stateManager.VisitedFolders))
assert.Equal(t, 1000, int(stateManager.DelayedFiles))
assert.Equal(t, 1000, int(stateManager.TransferFailures))
signedVisitedFolders, err := safeconvert.Uint64ToInt(stateManager.VisitedFolders)
assert.NoError(t, err)
assert.Equal(t, 1000, signedVisitedFolders)
signedDelayedFiles, err := safeconvert.Uint64ToInt(stateManager.DelayedFiles)
assert.NoError(t, err)
assert.Equal(t, 1000, signedDelayedFiles)
signedTransferFailures, err := safeconvert.Uint64ToInt(stateManager.TransferFailures)
assert.NoError(t, err)
assert.Equal(t, 1000, signedTransferFailures)

// Concurrently decrement delayed artifacts and transfer failures
for i := 0; i < 500; i++ {
Expand All @@ -356,6 +363,10 @@ func TestStateConcurrency(t *testing.T) {
wg.Wait()

// Assert 500 in delayed artifacts and transfer failures
assert.Equal(t, 500, int(stateManager.DelayedFiles))
assert.Equal(t, 500, int(stateManager.TransferFailures))
signedDelayedFiles, err = safeconvert.Uint64ToInt(stateManager.DelayedFiles)
assert.NoError(t, err)
assert.Equal(t, 500, signedDelayedFiles)
signedTransferFailures, err = safeconvert.Uint64ToInt(stateManager.TransferFailures)
assert.NoError(t, err)
assert.Equal(t, 500, signedTransferFailures)
}
51 changes: 33 additions & 18 deletions artifactory/commands/transferfiles/state/timeestimation.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package state

import (
"errors"
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"time"

"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
Expand Down Expand Up @@ -39,19 +41,23 @@ type TimeEstimationManager struct {
stateManager *TransferStateManager
}

func (tem *TimeEstimationManager) AddChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) {
func (tem *TimeEstimationManager) AddChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) error {
if durationMillis == 0 {
return
return nil
}

tem.addDataChunkStatus(chunkStatus, durationMillis)
return tem.addDataChunkStatus(chunkStatus, durationMillis)
}

func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) {
func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus, durationMillis int64) error {
var chunkSizeBytes int64
for _, file := range chunkStatus.Files {
if file.Status != api.Fail {
tem.CurrentTotalTransferredBytes += uint64(file.SizeBytes)
unsignedSizeBytes, err := safeconvert.Int64ToUint64(file.SizeBytes)
if err != nil {
return fmt.Errorf("failed to calculate the estimated remaining time: %w", err)
}
tem.CurrentTotalTransferredBytes += unsignedSizeBytes
}
if (file.Status == api.Success || file.Status == api.SkippedLargeProps) && !file.ChecksumDeployed {
chunkSizeBytes += file.SizeBytes
Expand All @@ -60,13 +66,13 @@ func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus

// If no files were uploaded regularly (with no errors and not checksum-deployed), don't use this chunk for the time estimation calculation.
if chunkSizeBytes == 0 {
return
return nil
}

workingThreads, err := tem.stateManager.GetWorkingThreads()
if err != nil {
log.Error("Couldn't calculate time estimation:", err.Error())
return
return err
}
speed := calculateChunkSpeed(workingThreads, chunkSizeBytes, durationMillis)
tem.LastSpeeds = append(tem.LastSpeeds, speed)
Expand All @@ -79,10 +85,11 @@ func (tem *TimeEstimationManager) addDataChunkStatus(chunkStatus api.ChunkStatus
}
if len(tem.LastSpeeds) == 0 {
tem.SpeedsAverage = 0
return
return err
}
// Calculate speed in bytes/ms
tem.SpeedsAverage = tem.LastSpeedsSum / float64(len(tem.LastSpeeds))
return nil
}

func calculateChunkSpeed(workingThreads int, chunkSizeSum, chunkDuration int64) float64 {
Expand All @@ -108,31 +115,39 @@ func (tem *TimeEstimationManager) GetSpeedString() string {
// 1. 5 minutes not passed since the beginning of the transfer
// 2. No files transferred
// 3. The transfer speed is less than 1 byte per second
func (tem *TimeEstimationManager) GetEstimatedRemainingTimeString() string {
remainingTimeSec := tem.getEstimatedRemainingSeconds()
if remainingTimeSec == 0 {
return "Not available yet"
func (tem *TimeEstimationManager) GetEstimatedRemainingTimeString() (string, error) {
remainingTimeSec, err := tem.getEstimatedRemainingSeconds()
if remainingTimeSec == 0 || err != nil {
return "Not available yet", err
}

return SecondsToLiteralTime(int64(remainingTimeSec), "About ")
signedRemainingTimeSec, err := safeconvert.Uint64ToInt64(remainingTimeSec)
if err != nil {
return "", errors.New("failed to calculate the estimated remaining time: " + err.Error())
}
return SecondsToLiteralTime(signedRemainingTimeSec, "About "), nil
}

func (tem *TimeEstimationManager) getEstimatedRemainingSeconds() uint64 {
func (tem *TimeEstimationManager) getEstimatedRemainingSeconds() (uint64, error) {
if tem.CurrentTotalTransferredBytes == 0 {
// No files transferred
return 0
return 0, nil
}
duration := time.Since(tem.stateManager.startTimestamp)
if duration < minTransferTimeToShowEstimation {
// 5 minutes not yet passed
return 0
return 0, nil
}

transferredBytesInSeconds := tem.CurrentTotalTransferredBytes / uint64(duration.Seconds())
if transferredBytesInSeconds == 0 {
// Less than 1 byte per second
return 0
return 0, nil
}
remainingBytes := tem.stateManager.OverallTransfer.TotalSizeBytes - tem.stateManager.OverallTransfer.TransferredSizeBytes
return uint64(remainingBytes) / transferredBytesInSeconds
unsignedRemainingBytes, err := safeconvert.Int64ToUint64(remainingBytes)
if err != nil {
return 0, fmt.Errorf("failed to calculate the estimated remaining time: %w", err)
}
return unsignedRemainingBytes / transferredBytesInSeconds, nil
}
40 changes: 29 additions & 11 deletions artifactory/commands/transferfiles/state/timeestimation_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package state

import (
"github.com/jfrog/gofrog/safeconvert"
"testing"
"time"

Expand Down Expand Up @@ -51,7 +52,9 @@ func TestGetSpeed(t *testing.T) {
addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
assert.Equal(t, 7.5, timeEstMng.getSpeed())
assert.Equal(t, "7.500 MB/s", timeEstMng.GetSpeedString())
assert.NotZero(t, timeEstMng.getEstimatedRemainingSeconds())
estimatedRemainingSeconds, err := timeEstMng.getEstimatedRemainingSeconds()
assert.NoError(t, err)
assert.NotZero(t, estimatedRemainingSeconds)

// Chunk 2: the upload of one of the files failed and the files are not included in the repository's total size (includedInTotalSize == false)
chunkStatus2 := api.ChunkStatus{
Expand All @@ -63,20 +66,27 @@ func TestGetSpeed(t *testing.T) {
addChunkStatus(t, timeEstMng, chunkStatus2, 2, false, 5*milliSecsInSecond)
assert.Equal(t, float64(8), timeEstMng.getSpeed())
assert.Equal(t, "8.000 MB/s", timeEstMng.GetSpeedString())
assert.NotZero(t, timeEstMng.getEstimatedRemainingSeconds())
estimatedRemainingSeconds, err = timeEstMng.getEstimatedRemainingSeconds()
assert.NoError(t, err)
assert.NotZero(t, estimatedRemainingSeconds)
}

func TestGetEstimatedRemainingSeconds(t *testing.T) {
timeEstMng, cleanUp := initTimeEstimationDataTest(t)
defer cleanUp()

timeEstMng.CurrentTotalTransferredBytes = uint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes)
unsignedTotalSizeBytes, err := safeconvert.Int64ToUint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes)
assert.NoError(t, err)
timeEstMng.CurrentTotalTransferredBytes = unsignedTotalSizeBytes
timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = timeEstMng.stateManager.OverallTransfer.TotalSizeBytes
assert.Zero(t, timeEstMng.getEstimatedRemainingSeconds())
estimatedRemainingSeconds, err := timeEstMng.getEstimatedRemainingSeconds()
assert.NoError(t, err)
assert.Zero(t, estimatedRemainingSeconds)

timeEstMng.CurrentTotalTransferredBytes = uint64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) / 2
timeEstMng.CurrentTotalTransferredBytes = unsignedTotalSizeBytes / 2
timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = timeEstMng.stateManager.OverallTransfer.TotalSizeBytes / 2
calculatedEstimatedSeconds := timeEstMng.getEstimatedRemainingSeconds()
calculatedEstimatedSeconds, err := timeEstMng.getEstimatedRemainingSeconds()
assert.NoError(t, err)
assert.NotZero(t, calculatedEstimatedSeconds)
}

Expand All @@ -90,7 +100,9 @@ func TestGetEstimatedRemainingTimeStringNotAvailableYet(t *testing.T) {
createFileUploadStatusResponse(repo1Key, 8*rtServicesUtils.SizeMiB, true, api.Success),
},
}
assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString())
estimatedRemainingTime, err := timeEstMng.GetEstimatedRemainingTimeString()
assert.NoError(t, err)
assert.Equal(t, "Not available yet", estimatedRemainingTime)
addChunkStatus(t, timeEstMng, chunkStatus1, 3, true, 10*milliSecsInSecond)
assert.Equal(t, "Not available yet", timeEstMng.GetSpeedString())
}
Expand All @@ -101,17 +113,23 @@ func TestGetEstimatedRemainingTimeString(t *testing.T) {

// Test "Not available yet" by setting the TotalTransferredBytes to 0
timeEstMng.CurrentTotalTransferredBytes = 0
assert.Equal(t, "Not available yet", timeEstMng.GetEstimatedRemainingTimeString())
estimatedRemainingTime, err := timeEstMng.GetEstimatedRemainingTimeString()
assert.NoError(t, err)
assert.Equal(t, "Not available yet", estimatedRemainingTime)

// Test "About 1 minute" by setting the transferred bytes to 80%
timeEstMng.CurrentTotalTransferredBytes = uint64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.8)
timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = int64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.8)
assert.Equal(t, "About 1 minute", timeEstMng.GetEstimatedRemainingTimeString())
estimatedRemainingTime, err = timeEstMng.GetEstimatedRemainingTimeString()
assert.NoError(t, err)
assert.Equal(t, "About 1 minute", estimatedRemainingTime)

// Test "Less than a minute" by setting the transferred bytes to 90%
timeEstMng.CurrentTotalTransferredBytes = uint64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.9)
timeEstMng.stateManager.OverallTransfer.TransferredSizeBytes = int64(float64(timeEstMng.stateManager.OverallTransfer.TotalSizeBytes) * 0.9)
assert.Equal(t, "Less than a minute", timeEstMng.GetEstimatedRemainingTimeString())
estimatedRemainingTime, err = timeEstMng.GetEstimatedRemainingTimeString()
assert.NoError(t, err)
assert.Equal(t, "Less than a minute", estimatedRemainingTime)
}

func newDefaultTimeEstimationManager(t *testing.T, buildInfoRepos bool) *TimeEstimationManager {
Expand Down Expand Up @@ -220,7 +238,7 @@ func addChunkStatus(t *testing.T, timeEstMng *TimeEstimationManager, chunkStatus
assert.NoError(t, err)
}
assert.NoError(t, timeEstMng.stateManager.SetWorkingThreads(workingThreads))
timeEstMng.AddChunkStatus(chunkStatus, durationMillis)
assert.NoError(t, timeEstMng.AddChunkStatus(chunkStatus, durationMillis))
}

func assertTransferredSizes(t *testing.T, stateManager *TransferStateManager, repo1expected, repo2expected int64) {
Expand Down
13 changes: 10 additions & 3 deletions artifactory/commands/transferfiles/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ func ShowStatus() error {
return nil
}

addOverallStatus(stateManager, &output, stateManager.GetRunningTimeString())
if err = addOverallStatus(stateManager, &output, stateManager.GetRunningTimeString()); err != nil {
return err
}
if stateManager.CurrentRepoKey != "" {
transferState, exists, err := state.LoadTransferState(stateManager.CurrentRepoKey, false)
if err != nil {
Expand All @@ -71,20 +73,25 @@ func isStopping() (bool, error) {
return fileutils.IsFileExists(filepath.Join(transferDir, StopFileName), false)
}

func addOverallStatus(stateManager *state.TransferStateManager, output *strings.Builder, runningTime string) {
func addOverallStatus(stateManager *state.TransferStateManager, output *strings.Builder, runningTime string) error {
addTitle(output, "Overall Transfer Status")
addString(output, coreutils.RemoveEmojisIfNonSupportedTerminal("🟒"), "Status", "Running", 3)
addString(output, "πŸƒ", "Running for", runningTime, 3)
addString(output, "πŸ—„ ", "Storage", sizeToString(stateManager.OverallTransfer.TransferredSizeBytes)+" / "+sizeToString(stateManager.OverallTransfer.TotalSizeBytes)+calcPercentageInt64(stateManager.OverallTransfer.TransferredSizeBytes, stateManager.OverallTransfer.TotalSizeBytes), 3)
addString(output, "πŸ“¦", "Repositories", fmt.Sprintf("%d / %d", stateManager.TotalRepositories.TransferredUnits, stateManager.TotalRepositories.TotalUnits)+calcPercentageInt64(stateManager.TotalRepositories.TransferredUnits, stateManager.TotalRepositories.TotalUnits), 2)
addString(output, "🧡", "Working threads", strconv.Itoa(stateManager.WorkingThreads), 2)
addString(output, "⚑", "Transfer speed", stateManager.GetSpeedString(), 2)
addString(output, "βŒ›", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1)
estimatedRemainingTime, err := stateManager.GetEstimatedRemainingTimeString()
if err != nil {
return err
}
addString(output, "βŒ›", "Estimated time remaining", estimatedRemainingTime, 1)
failureTxt := strconv.FormatUint(stateManager.TransferFailures, 10)
if stateManager.TransferFailures > 0 {
failureTxt += " (" + progressbar.RetryFailureContentNote + ")"
}
addString(output, "❌", "Transfer failures", failureTxt, 2)
return nil
}

func calcPercentageInt64(transferred, total int64) string {
Expand Down
Loading

0 comments on commit c92e0ab

Please sign in to comment.