feat:[AH-2541]: Add support for bulk skip artifacts (#147)

* adc10a removed unused variable and code

* f7442e added skip validation with API response data

* 87846f added skip validation with mock data

Author: cashwinichourasia

module/ar/migrate/adapter/adapter.go

@@ -54,6 +54,10 @@ type Adapter interface {
 		fileName *types.File,
 		artifactType types.ArtifactType,
 	) (bool, error)
+	GetAllFilesForVersion(
+		ctx context.Context,
+		registryRef, pkg, version string,
+	) ([]string, error)
 	CreateVersion(
 		registry string,
 		artifactName string,

module/ar/migrate/adapter/har/adapter.go

@@ -3,6 +3,7 @@ package har
 import (
 	"context"
 	"fmt"
+
 	//"github.com/harness/harness-cli/module/ar/migrate"
 	//client2 "github.com/harness/harness-cli/util/client"
 	"io"
@@ -170,6 +171,13 @@ func (a *adapter) FileExists(
 	return a.client.artifactFileExists(ctx, registryRef, pkg, version, file, artifactType)
 }
 
+func (a *adapter) GetAllFilesForVersion(
+	ctx context.Context,
+	registryRef, pkg, version string,
+) ([]string, error) {
+	return a.client.artifactGetFilesForVersion(ctx, registryRef, pkg, version)
+}
+
 func (a *adapter) CreateVersion(
 	registry string,
 	artifactName string,

module/ar/migrate/adapter/har/client.go

@@ -561,6 +561,41 @@ func (c *client) artifactFileExists(
 	return false, nil
 }
 
+func (c *client) artifactGetFilesForVersion(
+	ctx context.Context,
+	registryRef, pkg, version string,
+) ([]string, error) {
+	page := int64(0)
+	size := int64(100)
+
+	var allFileNames []string
+	for {
+		response, err := c.apiClient.GetArtifactFilesWithResponse(ctx, registryRef, pkg, version,
+			&ar.GetArtifactFilesParams{
+				Page:      &page,
+				Size:      &size,
+				SortOrder: nil,
+				SortField: nil,
+			})
+		if err != nil {
+			return allFileNames, fmt.Errorf("failed to get artifact files: %w", err)
+		}
+		if response.StatusCode() != http2.StatusOK {
+			return allFileNames, fmt.Errorf("failed to get artifact files: %s", response.Status())
+		}
+		data := response.JSON200
+
+		for _, v := range data.Data.Files {
+			allFileNames = append(allFileNames, v.Name)
+		}
+		if len(data.Data.Files) < int(size) || (nil != data.Data.PageCount && nil != data.Data.PageIndex && (*data.Data.PageIndex+1 >= *data.Data.PageCount)) {
+			break
+		}
+		page++
+	}
+	return allFileNames, nil
+}
+
 func (c *client) getRegistry(
 	ctx context.Context,
 	registry string,

module/ar/migrate/adapter/jfrog/adapter.go

@@ -768,6 +768,13 @@ func (a *adapter) FileExists(
 	return false, fmt.Errorf("not implemented")
 }
 
+func (a *adapter) GetAllFilesForVersion(
+	ctx context.Context,
+	registryRef, pkg, version string,
+) ([]string, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 type repomdData struct {
 	XMLName xml.Name `xml:"repomd"`
 	Data    []struct {

module/ar/migrate/adapter/mock_jfrog/adapter.go

@@ -829,6 +829,13 @@ func (a *adapter) FileExists(
 	return false, nil
 }
 
+func (a *adapter) GetAllFilesForVersion(
+	ctx context.Context,
+	registryRef, pkg, version string,
+) ([]string, error) {
+	return nil, nil
+}
+
 type repomdData struct {
 	XMLName xml.Name `xml:"repomd"`
 	Data    []struct {

module/ar/migrate/adapter/nexus/adapter.go

@@ -410,6 +410,13 @@ func (a *adapter) FileExists(
 	return false, nil
 }
 
+func (a *adapter) GetAllFilesForVersion(
+	ctx context.Context,
+	registryRef, pkg, version string,
+) ([]string, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 func (a *adapter) constructFilePath(pkg, version, fileName string, artifactType types.ArtifactType) string {
 	switch artifactType {
 	case types.MAVEN:

module/ar/migrate/migratable/file.go

@@ -23,7 +23,6 @@ import (
 	"github.com/harness/harness-cli/module/ar/migrate/engine"
 	"github.com/harness/harness-cli/module/ar/migrate/types"
 	"github.com/harness/harness-cli/module/ar/migrate/types/npm"
-	"github.com/harness/harness-cli/module/ar/migrate/util"
 	"github.com/harness/harness-cli/util/common"
 
 	"github.com/google/uuid"
@@ -115,31 +114,6 @@ func (r *File) Pre(ctx context.Context) error {
 	logger.Info().Msg("Starting version pre-migration step")
 	startTime := time.Now()
 
-	if !r.config.Overwrite && (r.artifactType != types.MAVEN && r.pkg.Name != "" && r.version.Name != "" && r.file.Name != "") {
-		exists, err := r.destAdapter.FileExists(ctx,
-			r.registry.Path,
-			r.pkg.Name, r.version.Name, r.file, r.artifactType)
-		if err != nil {
-			log.Error().Err(err).Msg("Failed to check if version exists")
-			return nil
-		}
-		if exists {
-			util.GetSkipPrinter().Println(fmt.Sprintf("Registry [%s], Package [%s/%s], File [%s] already exists",
-				r.destRegistry,
-				r.pkg.Name, r.version.Name, r.file.Name))
-			r.skipMigration = true
-			stat := types.FileStat{
-				Name:     r.file.Name,
-				Registry: r.srcRegistry,
-				Uri:      r.file.Uri,
-				Size:     int64(r.file.Size),
-				Status:   types.StatusSkip,
-			}
-			r.stats.FileStats = append(r.stats.FileStats, stat)
-			return nil
-		}
-	}
-
 	logger.Info().
 		Dur("duration", time.Since(startTime)).
 		Msg("Completed file pre-migration step")

module/ar/migrate/migratable/version.go

@@ -11,26 +11,28 @@ import (
 	"github.com/harness/harness-cli/module/ar/migrate/engine"
 	"github.com/harness/harness-cli/module/ar/migrate/tree"
 	"github.com/harness/harness-cli/module/ar/migrate/types"
+	"github.com/harness/harness-cli/module/ar/migrate/util"
 
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 )
 
 type Version struct {
-	srcRegistry  string
-	destRegistry string
-	srcAdapter   adapter.Adapter
-	destAdapter  adapter.Adapter
-	artifactType types.ArtifactType
-	logger       zerolog.Logger
-	pkg          types.Package
-	version      types.Version
-	node         *types.TreeNode
-	stats        *types.TransferStats
-	mapping      *types.RegistryMapping
-	config       *types.Config
-	registry     types.RegistryInfo
+	srcRegistry     string
+	destRegistry    string
+	srcAdapter      adapter.Adapter
+	destAdapter     adapter.Adapter
+	artifactType    types.ArtifactType
+	logger          zerolog.Logger
+	pkg             types.Package
+	version         types.Version
+	node            *types.TreeNode
+	stats           *types.TransferStats
+	mapping         *types.RegistryMapping
+	config          *types.Config
+	registry        types.RegistryInfo
+	existingFileMap map[string]bool
 }
 
 func NewVersionJob(
@@ -59,19 +61,20 @@ func NewVersionJob(
 		Logger()
 
 	return &Version{
-		srcRegistry:  srcRegistry,
-		destRegistry: destRegistry,
-		srcAdapter:   src,
-		destAdapter:  dest,
-		artifactType: artifactType,
-		logger:       jobLogger,
-		pkg:          pkg,
-		version:      version,
-		node:         node,
-		stats:        stats,
-		mapping:      mapping,
-		config:       config,
-		registry:     registry,
+		srcRegistry:     srcRegistry,
+		destRegistry:    destRegistry,
+		srcAdapter:      src,
+		destAdapter:     dest,
+		artifactType:    artifactType,
+		logger:          jobLogger,
+		pkg:             pkg,
+		version:         version,
+		node:            node,
+		stats:           stats,
+		mapping:         mapping,
+		config:          config,
+		registry:        registry,
+		existingFileMap: make(map[string]bool),
 	}
 }
 
@@ -89,6 +92,20 @@ func (r *Version) Pre(ctx context.Context) error {
 	logger.Info().Msg("Starting version pre-migration step")
 	startTime := time.Now()
 
+	// reading all existing files for this version from destination
+
+	if !r.config.Overwrite && (r.artifactType != types.MAVEN && r.pkg.Name != "" && r.version.Name != "") {
+		existingFiles, err := r.getAllExistingFilesForThisVersion(ctx)
+		if err != nil {
+			logger.Warn().Err(err).Msg("Failed to get existing files, will proceed with migration")
+		} else {
+			// Populate existingFileMap with file name
+			for _, fileName := range existingFiles {
+				r.existingFileMap[fileName] = true
+			}
+			logger.Info().Msgf("Found %d existing files for version %s", len(r.existingFileMap), r.version.Name)
+		}
+	}
 	logger.Info().
 		Dur("duration", time.Since(startTime)).
 		Msg("Completed version pre-migration step")
@@ -101,7 +118,6 @@ func (r *Version) Migrate(ctx context.Context) error {
 		Str("step", "migrate").
 		Str("trace_id", traceID).
 		Logger()
-
 	logger.Info().Msg("Starting version migration step")
 	startTime := time.Now()
 
@@ -115,6 +131,25 @@ func (r *Version) Migrate(ctx context.Context) error {
 			return fmt.Errorf("get files from tree failed: %w", err)
 		}
 		for _, file := range files {
+			// Check if file already exists in destination
+			if r.existingFileMap[file.Name] {
+				util.GetSkipPrinter().Println(fmt.Sprintf("Registry [%s], Package [%s/%s], File [%s] already exists",
+					r.destRegistry,
+					r.pkg.Name, r.version.Name, file.Name))
+				logger.Info().Msgf("Skipping file %s as it already exists in destination", file.Uri)
+
+				// Add to statistics
+				stat := types.FileStat{
+					Name:     file.Name,
+					Registry: r.srcRegistry,
+					Uri:      file.Uri,
+					Size:     int64(file.Size),
+					Status:   types.StatusSkip,
+				}
+				r.stats.FileStats = append(r.stats.FileStats, stat)
+				continue
+			}
+
 			job := NewFileJob(r.srcAdapter, r.destAdapter, r.srcRegistry, r.destRegistry, r.artifactType, r.pkg,
 				r.version, r.node, file, r.stats, r.mapping, r.config, r.registry)
 			jobs = append(jobs, job)
@@ -202,3 +237,22 @@ func (r *Version) Post(ctx context.Context) error {
 		Msg("Completed version post-migration step")
 	return nil
 }
+
+// getAllExistingFilesForThisVersion fetches existing files for this version from the destination API
+// Returns a slice of file paths that already exist
+func (r *Version) getAllExistingFilesForThisVersion(ctx context.Context) ([]string, error) {
+	// Call the destination adapter API to get all files for this version
+	allFileNames, err := r.destAdapter.GetAllFilesForVersion(
+		ctx,
+		r.registry.Path,
+		r.pkg.Name,
+		r.version.Name,
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to get existing files from destination: %w", err)
+	}
+
+	r.logger.Info().Msgf("Retrieved %d existing files from destination API for version %s", len(allFileNames), r.version.Name)
+	return allFileNames, nil
+}